E
Extremest
Here is the code I have so far. It connects to a db and grabs headers.
It then sorts them into groups and then puts all the complete ones
into another table. Problem I am having is that for some reason now it
is not finding ones that are single posts. Here is an example of a
header for a single.
(Ask the Dust ) [001/001] - "atd-ftc-repack.nfo" www.ctjes.com (1/1)
(1/1) at the end means it is part 1 of a 1 part post.
Any help would be greatly appreciated.
using System;
using System.Collections;
using System.Text;
using MySql.Data;
using System.Text.RegularExpressions;
namespace groupheaders
{
class Program
{
static Hashtable master;
static ArrayList full;
static Header[] narray;
static MySql.Data.MySqlClient.MySqlConnection conn = new
MySql.Data.MySqlClient.MySqlConnection();
static MySql.Data.MySqlClient.MySqlConnection connr = new
MySql.Data.MySqlClient.MySqlConnection();
static MySql.Data.MySqlClient.MySqlCommand cmd = new
MySql.Data.MySqlClient.MySqlCommand();
static MySql.Data.MySqlClient.MySqlCommand cmdi = new
MySql.Data.MySqlClient.MySqlCommand();
static MySql.Data.MySqlClient.MySqlCommand cmdu = new
MySql.Data.MySqlClient.MySqlCommand();
static MySql.Data.MySqlClient.MySqlCommand cmds = new
MySql.Data.MySqlClient.MySqlCommand();
static string myConnectionString =
"server=127.0.0.1;uid=root;pwd=pass;database=test;";
static string myConnectionStringr =
"server=local;uid=root;pwd=pass;database=db;";
//static string myConnectionStringr =
"server=127.0.0.1;uid=root;pwd=pass;database=test;";
static ArrayList groupget;
static string[] groups;
static Regex seg = new Regex("\\([0-9]*/[0-9]*\\)",
RegexOptions.IgnoreCase);
static string group;
static string table;
static Regex reg = new Regex("\\.");
static bool prepared = false;
static bool prepareds = false;
static int numbfound = 1;
class RelatedHeaders : IEnumerable
{
private string _realSubject = null;
private ArrayList _list = new ArrayList();
private int _max;
public RelatedHeaders(Header firstHeader)
{
this._realSubject = firstHeader.RealSubject;
this._max = firstHeader.MaxMessages;
this._list.Add(firstHeader);
}
public int max()
{
return this._max;
}
public int count()
{
return this._list.Count;
}
public void Add(Header newHeader)
{
if (newHeader.RealSubject != this._realSubject)
{
throw new ArgumentException(String.Format("New
header has subject '{0}', but should be '{1}'.", newHeader.RealSubject,
this._realSubject), "newHeader");
}
this._list.Add(newHeader);
//Console.WriteLine("{0}-{1}", this.count(),
this._max);
if (this.count() == this._max)
full.Add(this._list);
}
public ArrayList GetList()
{
return this._list;
}
public IEnumerator GetEnumerator()
{
return this._list.GetEnumerator();
}
}
class Header
{
private string numb;
private string subject;
private int messageNumber;
private int maxMessages;
private string realSubject;
private string date;
private string from;
private string msg_id;
private string bytes;
private string group;
public Header(string numb, string subject, string date,
string
from, string msg_id, string bytes, string groups)
{
this.numb = numb;
this.subject = subject;
this.date = date;
this.from = from;
this.msg_id = msg_id;
this.bytes = bytes;
this.group = groups;
ExtractMessageNumber(this.subject, out
this.messageNumber, out this.maxMessages, out this.realSubject);
this.realSubject = ExtractMainSubject(this.subject);
}
// Now pull apart the message title...
private void ExtractMessageNumber(string subject, out int
number, out int max, out string realsubject)
{
try
{
int end = subject.LastIndexOf(")");
int start = subject.LastIndexOf("(");
int length = (end) - (start + 1);
string fullset = subject.Substring(start + 1,
length);
string[] segments = fullset.Split('/');
number = int.Parse(segments[0]);
max = int.Parse(segments[1]);
realsubject = subject.Substring(0, start - 1);
}
catch (Exception e)
{
//Console.WriteLine(e);
number = 0;
max = 10;
realsubject = null;
}
}
private string ExtractMainSubject(string subject)
{
return seg.Replace(subject, "");
}
public string Number
{
get { return this.numb; }
}
public string Subject
{
get { return this.subject; }
}
public string Groups
{
get { return this.group; }
}
public int MessageNumber
{
get { return this.messageNumber; }
}
public int MaxMessages
{
get { return this.maxMessages; }
}
public string RealSubject
{
get { return this.realSubject; }
}
public string Date
{
get { return this.date; }
}
public string From
{
get { return this.from; }
}
public string Msg_id
{
get { return this.msg_id; }
}
public string Bytes
{
get { return this.bytes; }
}
}
static void Main(string[] args)
{
getgroups();
while (numbfound > 0)
{
numbfound = 0;
conn.ConnectionString = myConnectionString;
conn.Open();
connr.ConnectionString = myConnectionStringr;
connr.Open();
cmd.Connection = conn;
cmdu.Connection = conn;
cmds.Connection = connr;
cmdi.Connection = connr;
for (int x = 0; x < groups.Length; x++)
{
//int start = 0;
//do
//{
full = new ArrayList();
table = reg.Replace(groups[x], "");
group = groups[x];
getheaders();
Console.WriteLine("Master Has This Size {0}",
master.Count);
master.Clear();
Console.WriteLine("Have this many in {0} -
{1}", group, full.Count);
for (int y = 0; y < full.Count; y++)
{
fullfind((ArrayList)full[y]);
}
//start = start + 50000;
//} while (master.Count > 0);
prepared = false;
prepareds = false;
}
conn.Close();
connr.Close();
Console.WriteLine(numbfound);
int sleeptime = 5000;
//System.Threading.Thread.Sleep(sleeptime);
}
}
static void getgroups()
{
MySql.Data.MySqlClient.MySqlConnection conn2 = new
MySql.Data.MySqlClient.MySqlConnection();
MySql.Data.MySqlClient.MySqlCommand cmd2 = new
MySql.Data.MySqlClient.MySqlCommand();
string myConnectionString2 =
"server=127.0.0.1;uid=root;pwd=pass;database=test;";
conn2.ConnectionString = myConnectionString2;
conn2.Open();
cmd2.Connection = conn2;
cmd2.CommandText = "select * from `groups`";
MySql.Data.MySqlClient.MySqlDataReader reader1;
reader1 = cmd2.ExecuteReader();
groupget = new ArrayList();
while (reader1.Read())
{
string temp = reader1.GetValue(0).ToString();
groupget.Add(temp);
}
reader1.Close();
conn2.Close();
groups = new string[groupget.Count];
for (int x = 0; x < groupget.Count; x++)
{
groups[x] = groupget[x].ToString();
//Console.WriteLine(groups[x]);
}
}
static void fullfind(ArrayList p)
{
Header test = (Header)p[0];
narray = new Header[test.MaxMessages + 1];
for (int y = 0; y < p.Count; y++)
{
test = (Header)p[y];
//Console.WriteLine("{0}-{1}-{2}", test.Subject,
test.MessageNumber, test.MaxMessages);
if (test.MessageNumber < narray.Length)
narray[test.MessageNumber] = test;
}
find(narray);
}
static void find(Header[] list)
{
int count = 0;
foreach (object o in list)
{
if (o == null)
count++;
}
if (count == 1)
{
if (list[0] == null)
{
//Console.Write(".");
insert(list);
update(list);
numbfound++;
}
}
}
static void insert(Header[] found)
{
string msg_id = null;
string bytes = null;
int totalbytes = 0;
int id;
for (int x = 1; x < found.Length; x++)
{
msg_id += found[x].Msg_id + "|";
bytes += found[x].Bytes + "|";
totalbytes += int.Parse(found[x].Bytes);
}
if (!prepared)
{
cmdi.CommandText = "insert ignore into `files`
(`subject`,`from`,`date`,`msg_ids`,`bytes`,`totalbytes`,`groups`)
values (?subject, ?from, ?date, ?msg_ids, ?bytes, ?totalbytes,
?groups)";
cmdi.Prepare();
prepared = true;
}
cmdi.Parameters.Add("?subject", found[1].RealSubject);
cmdi.Parameters.Add("?from", found[1].From);
cmdi.Parameters.Add("?date", found[1].Date);
cmdi.Parameters.Add("?msg_ids", msg_id);
cmdi.Parameters.Add("?bytes", bytes);
cmdi.Parameters.Add("?totalbytes", totalbytes);
cmdi.Parameters.Add("?groups", found[1].Groups);
cmdi.ExecuteNonQuery();
if (!prepareds)
{
cmds.CommandText = "insert ignore into `subjects`
(`id`,`subject`,`totalbytes`,`date`,`groups`) values (last_insert_id(),
?subject, ?totalbytes, ?date, ?groups)";
cmds.Prepare();
prepareds = true;
}
cmds.Parameters.Add("?subject", found[1].RealSubject);
cmds.Parameters.Add("?totalbytes", totalbytes);
cmds.Parameters.Add("?date", found[1].Date);
cmds.Parameters.Add("?groups", found[1].Groups);
cmds.ExecuteNonQuery();
}
static void update(Header[] found)
{
for (int x = 1; x < found.Length; x++)
{
cmdu.CommandText = "delete from `" + table + "` where
`numb` = '" + found[x].Number + "'";
cmdu.ExecuteNonQuery();
}
}
static void getheaders()
{
cmd.CommandText = "select * from `" + table + "` where
subject like '%.nfo%' limit 500000";
MySql.Data.MySqlClient.MySqlDataReader reader;
reader = cmd.ExecuteReader();
master = new Hashtable();
while (reader.Read())
{
Header h = new Header(reader.GetValue(0).ToString(),
reader.GetValue(1).ToString(), reader.GetValue(3).ToString(),
reader.GetValue(2).ToString(), reader.GetValue(4).ToString(),
reader.GetValue(5).ToString(), reader.GetValue(6).ToString());
RelatedHeaders group =
(RelatedHeaders)master[h.RealSubject];
if (group == null)
{
group = new RelatedHeaders(h);
master.Add(h.RealSubject, group);
}
else
{
group.Add(h);
}
}
reader.Close();
}
}
}
It then sorts them into groups and then puts all the complete ones
into another table. Problem I am having is that for some reason now it
is not finding ones that are single posts. Here is an example of a
header for a single.
(Ask the Dust ) [001/001] - "atd-ftc-repack.nfo" www.ctjes.com (1/1)
(1/1) at the end means it is part 1 of a 1 part post.
Any help would be greatly appreciated.
using System;
using System.Collections;
using System.Text;
using MySql.Data;
using System.Text.RegularExpressions;
namespace groupheaders
{
class Program
{
static Hashtable master;
static ArrayList full;
static Header[] narray;
static MySql.Data.MySqlClient.MySqlConnection conn = new
MySql.Data.MySqlClient.MySqlConnection();
static MySql.Data.MySqlClient.MySqlConnection connr = new
MySql.Data.MySqlClient.MySqlConnection();
static MySql.Data.MySqlClient.MySqlCommand cmd = new
MySql.Data.MySqlClient.MySqlCommand();
static MySql.Data.MySqlClient.MySqlCommand cmdi = new
MySql.Data.MySqlClient.MySqlCommand();
static MySql.Data.MySqlClient.MySqlCommand cmdu = new
MySql.Data.MySqlClient.MySqlCommand();
static MySql.Data.MySqlClient.MySqlCommand cmds = new
MySql.Data.MySqlClient.MySqlCommand();
static string myConnectionString =
"server=127.0.0.1;uid=root;pwd=pass;database=test;";
static string myConnectionStringr =
"server=local;uid=root;pwd=pass;database=db;";
//static string myConnectionStringr =
"server=127.0.0.1;uid=root;pwd=pass;database=test;";
static ArrayList groupget;
static string[] groups;
static Regex seg = new Regex("\\([0-9]*/[0-9]*\\)",
RegexOptions.IgnoreCase);
static string group;
static string table;
static Regex reg = new Regex("\\.");
static bool prepared = false;
static bool prepareds = false;
static int numbfound = 1;
class RelatedHeaders : IEnumerable
{
private string _realSubject = null;
private ArrayList _list = new ArrayList();
private int _max;
public RelatedHeaders(Header firstHeader)
{
this._realSubject = firstHeader.RealSubject;
this._max = firstHeader.MaxMessages;
this._list.Add(firstHeader);
}
public int max()
{
return this._max;
}
public int count()
{
return this._list.Count;
}
public void Add(Header newHeader)
{
if (newHeader.RealSubject != this._realSubject)
{
throw new ArgumentException(String.Format("New
header has subject '{0}', but should be '{1}'.", newHeader.RealSubject,
this._realSubject), "newHeader");
}
this._list.Add(newHeader);
//Console.WriteLine("{0}-{1}", this.count(),
this._max);
if (this.count() == this._max)
full.Add(this._list);
}
public ArrayList GetList()
{
return this._list;
}
public IEnumerator GetEnumerator()
{
return this._list.GetEnumerator();
}
}
class Header
{
private string numb;
private string subject;
private int messageNumber;
private int maxMessages;
private string realSubject;
private string date;
private string from;
private string msg_id;
private string bytes;
private string group;
public Header(string numb, string subject, string date,
string
from, string msg_id, string bytes, string groups)
{
this.numb = numb;
this.subject = subject;
this.date = date;
this.from = from;
this.msg_id = msg_id;
this.bytes = bytes;
this.group = groups;
ExtractMessageNumber(this.subject, out
this.messageNumber, out this.maxMessages, out this.realSubject);
this.realSubject = ExtractMainSubject(this.subject);
}
// Now pull apart the message title...
private void ExtractMessageNumber(string subject, out int
number, out int max, out string realsubject)
{
try
{
int end = subject.LastIndexOf(")");
int start = subject.LastIndexOf("(");
int length = (end) - (start + 1);
string fullset = subject.Substring(start + 1,
length);
string[] segments = fullset.Split('/');
number = int.Parse(segments[0]);
max = int.Parse(segments[1]);
realsubject = subject.Substring(0, start - 1);
}
catch (Exception e)
{
//Console.WriteLine(e);
number = 0;
max = 10;
realsubject = null;
}
}
private string ExtractMainSubject(string subject)
{
return seg.Replace(subject, "");
}
public string Number
{
get { return this.numb; }
}
public string Subject
{
get { return this.subject; }
}
public string Groups
{
get { return this.group; }
}
public int MessageNumber
{
get { return this.messageNumber; }
}
public int MaxMessages
{
get { return this.maxMessages; }
}
public string RealSubject
{
get { return this.realSubject; }
}
public string Date
{
get { return this.date; }
}
public string From
{
get { return this.from; }
}
public string Msg_id
{
get { return this.msg_id; }
}
public string Bytes
{
get { return this.bytes; }
}
}
static void Main(string[] args)
{
getgroups();
while (numbfound > 0)
{
numbfound = 0;
conn.ConnectionString = myConnectionString;
conn.Open();
connr.ConnectionString = myConnectionStringr;
connr.Open();
cmd.Connection = conn;
cmdu.Connection = conn;
cmds.Connection = connr;
cmdi.Connection = connr;
for (int x = 0; x < groups.Length; x++)
{
//int start = 0;
//do
//{
full = new ArrayList();
table = reg.Replace(groups[x], "");
group = groups[x];
getheaders();
Console.WriteLine("Master Has This Size {0}",
master.Count);
master.Clear();
Console.WriteLine("Have this many in {0} -
{1}", group, full.Count);
for (int y = 0; y < full.Count; y++)
{
fullfind((ArrayList)full[y]);
}
//start = start + 50000;
//} while (master.Count > 0);
prepared = false;
prepareds = false;
}
conn.Close();
connr.Close();
Console.WriteLine(numbfound);
int sleeptime = 5000;
//System.Threading.Thread.Sleep(sleeptime);
}
}
static void getgroups()
{
MySql.Data.MySqlClient.MySqlConnection conn2 = new
MySql.Data.MySqlClient.MySqlConnection();
MySql.Data.MySqlClient.MySqlCommand cmd2 = new
MySql.Data.MySqlClient.MySqlCommand();
string myConnectionString2 =
"server=127.0.0.1;uid=root;pwd=pass;database=test;";
conn2.ConnectionString = myConnectionString2;
conn2.Open();
cmd2.Connection = conn2;
cmd2.CommandText = "select * from `groups`";
MySql.Data.MySqlClient.MySqlDataReader reader1;
reader1 = cmd2.ExecuteReader();
groupget = new ArrayList();
while (reader1.Read())
{
string temp = reader1.GetValue(0).ToString();
groupget.Add(temp);
}
reader1.Close();
conn2.Close();
groups = new string[groupget.Count];
for (int x = 0; x < groupget.Count; x++)
{
groups[x] = groupget[x].ToString();
//Console.WriteLine(groups[x]);
}
}
static void fullfind(ArrayList p)
{
Header test = (Header)p[0];
narray = new Header[test.MaxMessages + 1];
for (int y = 0; y < p.Count; y++)
{
test = (Header)p[y];
//Console.WriteLine("{0}-{1}-{2}", test.Subject,
test.MessageNumber, test.MaxMessages);
if (test.MessageNumber < narray.Length)
narray[test.MessageNumber] = test;
}
find(narray);
}
static void find(Header[] list)
{
int count = 0;
foreach (object o in list)
{
if (o == null)
count++;
}
if (count == 1)
{
if (list[0] == null)
{
//Console.Write(".");
insert(list);
update(list);
numbfound++;
}
}
}
static void insert(Header[] found)
{
string msg_id = null;
string bytes = null;
int totalbytes = 0;
int id;
for (int x = 1; x < found.Length; x++)
{
msg_id += found[x].Msg_id + "|";
bytes += found[x].Bytes + "|";
totalbytes += int.Parse(found[x].Bytes);
}
if (!prepared)
{
cmdi.CommandText = "insert ignore into `files`
(`subject`,`from`,`date`,`msg_ids`,`bytes`,`totalbytes`,`groups`)
values (?subject, ?from, ?date, ?msg_ids, ?bytes, ?totalbytes,
?groups)";
cmdi.Prepare();
prepared = true;
}
cmdi.Parameters.Add("?subject", found[1].RealSubject);
cmdi.Parameters.Add("?from", found[1].From);
cmdi.Parameters.Add("?date", found[1].Date);
cmdi.Parameters.Add("?msg_ids", msg_id);
cmdi.Parameters.Add("?bytes", bytes);
cmdi.Parameters.Add("?totalbytes", totalbytes);
cmdi.Parameters.Add("?groups", found[1].Groups);
cmdi.ExecuteNonQuery();
if (!prepareds)
{
cmds.CommandText = "insert ignore into `subjects`
(`id`,`subject`,`totalbytes`,`date`,`groups`) values (last_insert_id(),
?subject, ?totalbytes, ?date, ?groups)";
cmds.Prepare();
prepareds = true;
}
cmds.Parameters.Add("?subject", found[1].RealSubject);
cmds.Parameters.Add("?totalbytes", totalbytes);
cmds.Parameters.Add("?date", found[1].Date);
cmds.Parameters.Add("?groups", found[1].Groups);
cmds.ExecuteNonQuery();
}
static void update(Header[] found)
{
for (int x = 1; x < found.Length; x++)
{
cmdu.CommandText = "delete from `" + table + "` where
`numb` = '" + found[x].Number + "'";
cmdu.ExecuteNonQuery();
}
}
static void getheaders()
{
cmd.CommandText = "select * from `" + table + "` where
subject like '%.nfo%' limit 500000";
MySql.Data.MySqlClient.MySqlDataReader reader;
reader = cmd.ExecuteReader();
master = new Hashtable();
while (reader.Read())
{
Header h = new Header(reader.GetValue(0).ToString(),
reader.GetValue(1).ToString(), reader.GetValue(3).ToString(),
reader.GetValue(2).ToString(), reader.GetValue(4).ToString(),
reader.GetValue(5).ToString(), reader.GetValue(6).ToString());
RelatedHeaders group =
(RelatedHeaders)master[h.RealSubject];
if (group == null)
{
group = new RelatedHeaders(h);
master.Add(h.RealSubject, group);
}
else
{
group.Add(h);
}
}
reader.Close();
}
}
}