c# - Is that necessary to dispose objects inside static functions? -
ok having major problem atm.
my software using extremely high amount of ram. using lot of htmlagilitypack.htmldocument objects big size pages sources.
however of objects used inside static functions , htmlagilitypack.htmldocument isn't idisposable
so need set every variable explicitly null ?
even if inside static functions ?
for example need set these variables null @ end of function below
the variables asking : lstdrwlist ? or since inside disposed automatically ?
should call explicitly garbage collector ?
c# .net 4.5 wpf application
private static void func_checkwaitingtoprocesspages(object state) { paralleloptions myoptions = new paralleloptions(); myoptions.maxdegreeofparallelism = publicsettings.ir_how_many_tasks_for_per_pages_process; list<datarow> lstdrwlist = new list<datarow>(); using (datatable dtmytable = dbconnection.db_select_datatable(srselecttopprocesspagesquery)) { foreach (datarow drw in dtmytable.rows) { lstdrwlist.add(drw); } } parallel.foreach(lstdrwlist, myoptions, drw => { process_given_page(drw); }); } the problem found issue how fix
here problem happens in 10 seconds used visual studio profiler
here full class causes huge memory leak issue
using htmlagilitypack; using system; using system.collections.generic; using system.data; using system.linq; using system.text; using system.threading; using system.threading.tasks; namespace doktora_tez_projesi_crawler_program { public static class pagesprocessor { private static timer _timer; private static int howmanyseconds = 10; public static void func_startcrawlingwaitingurls() { publicstaticfunctions.addmsgtoevents("checking waiting process crawled urls process started every " + howmanyseconds + " seconds!"); _timer = new timer(func_checkwaitingtoprocesspages, null, publicsettings.irtimers_delayed_start_miliseconds, howmanyseconds * 1000); } private static string srselecttopprocesspagesquery = " select top 100 cl_idurl,cl_roositeid,cl_crawlsource,cl_crawlorgurl tblcrawlurls " + " cl_pageprocessed=0 , cl_totalcrawltimes > 0 " + " order cl_lastprocessdate asc"; private static void func_checkwaitingtoprocesspages(object state) { paralleloptions myoptions = new paralleloptions(); myoptions.maxdegreeofparallelism = publicsettings.ir_how_many_tasks_for_per_pages_process; list<datarow> lstdrwlist = new list<datarow>(); using (datatable dtmytable = dbconnection.db_select_datatable(srselecttopprocesspagesquery)) { foreach (datarow drw in dtmytable.rows) { lstdrwlist.add(drw); } } parallel.foreach(lstdrwlist, myoptions, drw => { process_given_page(drw); }); } private class csproductfeatures { public string srproductrootsiteid = "null", srproducttitle = "null", srproductcode = "null", srproductimagelink = "null"; public string srproductdetailedexplanation = "null", srproductfeatures = "null", srcrawledorgurl = "null", srproductidcode = "null"; public bool blpossibleproductpage = false, blfreecargo = false, blproductpage = true; public list<string> lstproductcategories = new list<string>(); public int irproductprice = 0; public list<csproductcomments> lstproductcomments = new list<csproductcomments>(); public list<keyvaluepair<string, string>> lstproductfeatures = new list<keyvaluepair<string, string>>(); } private class csproductcomments { public string srcommenttitle = "null", srcommentpros = "null", srcommentcons = "null"; public int ircommentscore = 0; //0 = negative 5=full star } private static void process_given_page(datarow drw) { csproductfeatures temp_productfeatures = new csproductfeatures(); temp_productfeatures.srproductrootsiteid = drw["cl_roositeid"].tostring(); temp_productfeatures.srcrawledorgurl = drw["cl_crawlorgurl"].tostring(); htmldocument hdmydoc = new htmldocument();//nulled hdmydoc.loadhtml(drw["cl_crawlsource"].tostring()); bool blbreakloop = false; foreach (var vrvariable in publicvariables.dicrootsites[temp_productfeatures.srproductrootsiteid].lstrootsiteidentifiers) { if (vrvariable.srhtmlobjecttype != "link") { htmlnodecollection hdnodes; if (vrvariable.blselectmultiplenodes == false) hdnodes = hdmydoc.documentnode.selectnodes(string.format("//{0}[@{1}='{2}']", vrvariable.srhtmlobjecttype, vrvariable.srhtmlobjecttypeidentifier, vrvariable.srhtmlobjecttypename)); else hdnodes = hdmydoc.documentnode.selectnodes(string.format("//{0}[@{1}='{2}']//{3}", vrvariable.srhtmlobjecttype, vrvariable.srhtmlobjecttypeidentifier, vrvariable.srhtmlobjecttypename, vrvariable.srhtmlsubidentifiertype)); if (hdnodes == null && vrvariable.srindetifiertype == "producttitle") { blbreakloop = true; temp_productfeatures.blproductpage = false; continue; } if (blbreakloop == true) break; if (hdnodes == null) continue; string sr_node_required_val = "null"; if (hdnodes[0].innertext != null) sr_node_required_val = hdnodes[0].innertext; string srlinkval = "null"; if (vrvariable.srhtmlobjecttype == "a" && hdnodes[0].attributes != null) { if (hdnodes[0].attributes["href"] != null) { srlinkval = publicstaticfunctions.return_absolute_url(hdnodes[0].attributes["href"].value, temp_productfeatures.srcrawledorgurl); } } if (vrvariable.blgetvalue == true) { if (hdnodes[0].attributes != null) if (hdnodes[0].attributes["value"] != null) sr_node_required_val = hdnodes[0].attributes["value"].value; } sr_node_required_val = sr_node_required_val.trim(); switch (vrvariable.srindetifiertype) { case "productpage": temp_productfeatures.blpossibleproductpage = true; break; case "producttitle": temp_productfeatures.srproducttitle = sr_node_required_val; break; case "productcode": temp_productfeatures.srproductcode = sr_node_required_val; break; case "productcargo": temp_productfeatures.blfreecargo = true; break; case "productcategories": temp_productfeatures.lstproductcategories = func_return_product_categories(hdnodes); break; case "productprice": temp_productfeatures.irproductprice = func_return_product_price(sr_node_required_val, temp_productfeatures.srproductrootsiteid); break; case "productimage": temp_productfeatures.srproductimagelink = srlinkval; break; case "productidcode": temp_productfeatures.srproductidcode = sr_node_required_val; break; } } if (vrvariable.srhtmlobjecttype == "link") { string srlinktofetch = vrvariable.srhtmlobjecttypeidentifier; if (vrvariable.blusesproductidcode == true) { srlinktofetch = string.format(srlinktofetch, temp_productfeatures.srproductidcode); } string srfetchresult = crawlgivenurl.func_fetch_page(srlinktofetch); string srresulttoassign = "null"; if (srfetchresult == publicsettings.srcrawlfailedmessage) { srresulttoassign = srfetchresult; } else { htmldocument temp_hddocument = new htmldocument();//nulled temp_hddocument.loadhtml(srfetchresult); if (temp_hddocument.documentnode != null) if (temp_hddocument.documentnode.innertext != null) srresulttoassign = temp_hddocument.documentnode.innertext; temp_hddocument = null; } switch (vrvariable.srindetifiertype) { case "productexplanation": temp_productfeatures.srproductdetailedexplanation = srresulttoassign; break; case "productfeatures": temp_productfeatures.lstproductfeatures = func_return_product_features(temp_productfeatures.srproductrootsiteid, srfetchresult, temp_productfeatures.srcrawledorgurl); break; } } } if (temp_productfeatures.blproductpage == true) { string asdas = ""; } hdmydoc = null; } private static list<string> func_return_product_categories(htmlnodecollection hdnodecollection) { list<string> lstcategories = new list<string> { }; foreach (htmlnode hdnode in hdnodecollection) { if (hdnode.innertext != null) { lstcategories.add(hdnode.innertext); } } return lstcategories; } private static int func_return_product_price(string srpricetext, string srrootsiteid) { int irprice = 0; srpricetext = srpricetext.replace(publicvariables.dicrootsites[srrootsiteid].srpricedelimeter, ""); if (srpricetext.contains(publicvariables.dicrootsites[srrootsiteid].srpriceignoredelimeter) == true) { srpricetext = srpricetext.substring(0, srpricetext.indexof(publicvariables.dicrootsites[srrootsiteid].srpriceignoredelimeter)); } int32.tryparse(srpricetext, out irprice); return irprice; } private static list<keyvaluepair<string, string>> func_return_product_features(string srrootsiteid, string srpagesource, string srcrawlurl) { list<keyvaluepair<string, string>> lstfoundfeatures = new list<keyvaluepair<string, string>>(); if (srpagesource == publicsettings.srcrawlfailedmessage) return lstfoundfeatures; htmldocument temp_hddocument = new htmldocument();//nulled temp_hddocument.loadhtml(srpagesource); list<string> lstfeaturetitles = new list<string>(); list<string> lstfeaturedescriptions = new list<string>(); foreach (var vrvariable in publicvariables.dicrootsites[srrootsiteid].lstrootsitesfeaturesidentifiers) { if (vrvariable.blperfeatureidentifier == true) { htmlnodecollection hdnodes = temp_hddocument.documentnode.selectnodes(string.format("//{0}[@{1}='{2}']", vrvariable.srhtmlobjecttype, vrvariable.srhtmlobjectidentifier, vrvariable.srhtmlobjectidentifiername)); if (hdnodes != null) foreach (var vrnewvariable in publicvariables.dicrootsites[srrootsiteid].lstrootsitesfeaturesidentifiers) { if (vrnewvariable.blperfeatureidentifier == false) { foreach (htmlnode hdtempnode in hdnodes) { var vrtempnewnode = hdtempnode.selectsinglenode(string.format("//{0}[@{1}='{2}']", vrvariable.srhtmlobjecttype, vrvariable.srhtmlobjectidentifier, vrvariable.srhtmlobjectidentifiername)); if (vrtempnewnode != null) if (vrtempnewnode.innertext != null) { string srnodefeature = vrtempnewnode.innertext.trim(); switch (vrvariable.srwhichfeatureidentifier) { case "featuretitle": lstfeaturetitles.add(srnodefeature); break; case "featuredescription": lstfeaturedescriptions.add(srnodefeature); break; } } } } } break; } } temp_hddocument = null; if (lstfeaturedescriptions.count != lstfeaturetitles.count) { errorlogger.logerror("found features count not equal features description count crawled url: " + srcrawlurl); return lstfoundfeatures; } (int = 0; < lstfeaturedescriptions.count; i++) { keyvaluepair<string, string> mykeyvalpair = new keyvaluepair<string, string>(lstfeaturetitles[i], lstfeaturedescriptions[i]); lstfoundfeatures.add(mykeyvalpair); } return lstfoundfeatures; } } }
no, don't need set variables null in both static , instance methods. variables inside method (even inside static method) on stack space of method, go out of scope @ end of method execution , targeted garbage collection. , explicitly calling garbage collector isn't practice.
Comments
Post a Comment