From 9d5b50919952ba8510065572165d9a791190251e Mon Sep 17 00:00:00 2001 From: coulisse Date: Sat, 30 Mar 2024 06:45:14 +0100 Subject: [PATCH] issue #58 --- CITATION.cff | 2 +- docs/CHANGELOG.md | 3 +- lib/adxo.py | 140 +++++++++++++++++----------------- requirements.txt | 2 + scripts/build.sh | 4 +- scripts/dxcluster.db | 0 static/css/dev/style.css | 2 +- static/js/dev/index_inline.js | 2 +- static/js/dev/table.js | 5 +- static/js/rel/table.min.js | 2 +- webapp.py | 2 +- 11 files changed, 85 insertions(+), 79 deletions(-) create mode 100644 scripts/dxcluster.db diff --git a/CITATION.cff b/CITATION.cff index e13500b..ea2cbdf 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -25,4 +25,4 @@ keywords: - spiderweb license: GPL-3.0 version: v2.5.3 -date-released: 2024-03-24 +date-released: 2024-03-29 diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 53f97e4..e7205d6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,5 +1,5 @@ ### Change log -Date: 24/03/2024 +Date: 29/03/2024 Release: v2.5.3 - tested with Python 3.12 - replaced mysql driver with mariadb driver @@ -8,6 +8,7 @@ Release: v2.5.3 - upgraded bootstrap to 5.3.3 - issue [#51](https://github.com/coulisse/spiderweb/issues/51): added total number of users & nodes connected Issue - issue [#56](https://github.com/coulisse/spiderweb/issues/56): added a simple counter +- issue [#58](https://github.com/coulisse/spiderweb/issues/58) ___ Date: 10/03/2024 diff --git a/lib/adxo.py b/lib/adxo.py index d386d74..e6404a5 100644 --- a/lib/adxo.py +++ b/lib/adxo.py @@ -1,86 +1,90 @@ # *********************************************************************************** -# Module used to get Announced DX Operation from NG3K website via .ICS (Calendar) +# Module used to get Announced DX Operation from NG3K website via rss feed # file, parse it and return a dictionary with these events # *********************************************************************************** __author__ = "IU1BOW - Corrado" import requests import logging from datetime import datetime -import tempfile - -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s]: %(message)s", - datefmt="%m/%d/%Y %I:%M:%S", -) -# format single line -def format_line(prop): - prop_out = dict() - try: - dtstart = datetime.strptime(prop["DTSTART;VALUE=DATE"], "%Y%m%d") - dtend = datetime.strptime(prop["DTEND;VALUE=DATE"], "%Y%m%d") - now = datetime.now() - if dtstart <= now and dtend >= now: - prop_out["start"] = dtstart.strftime("%Y-%m-%dT%H:%M:%S%z") - prop_out["end"] = dtend.strftime("%Y-%m-%dT%H:%M:%S%z") - prop_out["summary"] = prop["SUMMARY"].split("(")[0].strip() - prop_out["callsign"] = prop["SUMMARY"].split("(", 1)[1].split(")", 1)[0] - prop_out["description"] = prop["DESCRIPTION"].replace("\\", "") - - except KeyError: - pass - - return prop_out - - -# TODO: url from conf parameter +import requests +import feedparser +import re +import unicodedata +def remove_control_characters(s): + return "".join(ch for ch in s if unicodedata.category(ch)[0]!="C") def get_adxo_events(): - url = "http://dxcal.kj4z.com/dxcal" - line_num = 0 - event_num = 0 + # URL del file XML RSS + rss_url = "https://www.ng3k.com/adxo.xml" + try: - logging.info("connection to: " + url) - req = requests.get(url) + + # download XML + response = requests.get(rss_url) + xml_content = response.content + + # parse XML + feed = feedparser.parse(xml_content) events = [] prop = dict() - prop_name = "" - with tempfile.TemporaryFile() as temp: - temp.write(req.content) - temp.seek(0) - lines = temp.readlines() - for line_bytes in lines: - line = line_bytes.decode() - line_num += 1 - current_line_array = line.strip().split(":", 1) - if current_line_array[0] == "BEGIN": - if current_line_array[1] == "VCALENDAR": - prop = {} - if current_line_array[1] == "VEVENT": - event_num += 1 - prop = {} - else: - if current_line_array[0] == "END": - if current_line_array[1] == "VCALENDAR": - pass - if current_line_array[1] == "VEVENT": - prop = format_line(prop) - if prop: - events.append(prop) - else: - if len(current_line_array) > 1: - prop_name = current_line_array[0] - prop[prop_name] = current_line_array[1] - else: - if len(prop_name) > 0: - prop[prop_name] = ( - prop[prop_name] + current_line_array[0] - ) + now = datetime.now() + + # extract elements + for item in feed.entries: + prop = {} + title = item.title + #title = "Sint Maarten: Dec 2 2023 - Jan 20 2024 -- PJ7AA -- QSL via: LoTW " + logging.debug(title) + + #callsign + start_callsign_idx = title.find("--") + end_callsign_idx = title.find("--", start_callsign_idx + 2) + prop["callsign"] = title[start_callsign_idx + 2:end_callsign_idx].strip() + + #period + period = title[title.find(":")+1: start_callsign_idx] + comma_year_idx = period.find(",") + + #start date - end date + if comma_year_idx > 0: + #Mar 23-Apr 1, 2024 or Mar 23-30, 2024 + year = period[comma_year_idx+1:].strip() + date_start = period[:period.find("-")]+" "+year + date_end = period[period.find("-")+1:comma_year_idx]+" "+year + match = re.search(r"^([A-Za-z]{3}) \d{1,2} \d{4}$", date_end) + if match: + #Mar 23-Apr 1, 2024 + pass + else: + #Mar 23-30, 2024 + date_end=date_start[:5]+date_end + else: + #Mar 23 2023-Apr 1 2024 + date_start = period[:period.find("-")] + date_end = period[period.find("-")+1:] + + prop["start"] = datetime.strptime(date_start.strip(), "%b %d %Y") + prop["end"] = datetime.strptime(date_end.strip(), "%b %d %Y") + + prop["summary"] = remove_control_characters(title) + prop["description"] = remove_control_characters(item.description) + + logging.debug("date start: "+ str(prop["start"]) ) + logging.debug("date end: "+ str(prop["end"]) ) + + #append only valids (in date) events + if prop["start"] <= now and prop["end"] >= now: + events.append(prop) + + logging.debug(events) + if len(events) > 0: + logging.info("number ADXO events: " + str(len(events))) + else: + logging.warn("No ADXO events founds") - logging.debug("number of line reads: " + str(line_num)) - logging.info("number ADXO events: " + str(event_num)) return events + except Exception as e1: logging.error(e1) return diff --git a/requirements.txt b/requirements.txt index 466dcb0..2a28fd1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ blinker==1.7.0 charset-normalizer==3.3.2 click==8.1.7 +feedparser==6.0.11 Flask==3.0.2 Flask-Minify==0.42 Flask-WTF==1.2.1 @@ -21,6 +22,7 @@ pytz==2024.1 rcssmin==1.1.2 requests==2.31.0 setuptools==68.2.2 +sgmllib3k==1.0.0 six==1.16.0 tzdata==2024.1 urllib3==2.2.1 diff --git a/scripts/build.sh b/scripts/build.sh index f022cbc..0d2c941 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -277,7 +277,7 @@ if [ "$2" == "-c" ]; then head -10 ../docs/CHANGELOG.md - read -p "Do you want to proceed to commit version ${ver} (yes/no) " yn + read -r -p "Do you want to proceed to commit version ${ver} (yes/no) " yn case $yn in yes ) echo ok, we will proceed;; @@ -297,7 +297,7 @@ if [ "$2" == "-c" ]; then fi echo 'Please, add comment for commit on tag ' ${ver} - read comm_tag_msg + read -r comm_tag_msg if ! git commit -m "${comm_tag_msg}"; then echo 'Error on commit' exit 9 diff --git a/scripts/dxcluster.db b/scripts/dxcluster.db new file mode 100644 index 0000000..e69de29 diff --git a/static/css/dev/style.css b/static/css/dev/style.css index 57bf611..85da21e 100644 --- a/static/css/dev/style.css +++ b/static/css/dev/style.css @@ -170,7 +170,7 @@ span.search-callsign { .kpi-card { overflow: hidden; position: relative; - box-shadow: 1px 1px 3px rgba(0,0,0,0.75);; + box-shadow: 1px 1px 3px rgba(0,0,0,0.75); display: inline-block; padding: 1em; border-radius: 0; diff --git a/static/js/dev/index_inline.js b/static/js/dev/index_inline.js index 248d2e2..1d6e351 100644 --- a/static/js/dev/index_inline.js +++ b/static/js/dev/index_inline.js @@ -6,7 +6,7 @@ //var my_adxo_events=jQuery.parseJSON(my_adxo_events_json.replaceAll("\t","")); var my_adxo_events = JSON.parse(my_adxo_events_json.replaceAll('\t', '')); - +//var my_adxo_events = JSON.parse(my_adxo_events_json) refresh_timer(); //run first data fetch var myRefresh = setInterval(refresh_timer, timer_interval_json); window.onload = () => { diff --git a/static/js/dev/table.js b/static/js/dev/table.js index 3d239df..6b57f47 100644 --- a/static/js/dev/table.js +++ b/static/js/dev/table.js @@ -320,9 +320,8 @@ function compose_filter(id, len, qry_json) { } } catch (err) { - if (err.name == 'TypeError') { - console.error(err.name); - /* error managed: it is ok: probabilly ther is no filter on cq region */ + if (err instanceof TypeError) { + console.log(err.name + ' managed - it is ok: probabilly ther is no filter on cq region'); } else { throw err; } diff --git a/static/js/rel/table.min.js b/static/js/rel/table.min.js index 8361f06..1a297fb 100644 --- a/static/js/rel/table.min.js +++ b/static/js/rel/table.min.js @@ -1 +1 @@ -class table_builder{constructor(e){this.selector=e,this.current_data=[],this.first_time=!0}getLastRowId(){let e;return e=null==this.current_data||this.current_data.length<1?0:this.current_data[0].rowid}resetData(){this.current_data=[]}#buildRow(e,t,n,a=""){var r=document.createElement("tr"),t=(0NG3K Website",l=document.createElement("i"),o=(l.className="bi-search",l.role="button",l.ariaLabel=e.dx,document.createElement("a")),s=(o.href=qrz_url+e.dx,o.target="_blank",o.rel="noopener",document.createElement("span")),c=document.createElement("mark"),c=(c.textContent=e.dx,e.dx==a?s.appendChild(c):s.textContent=" "+e.dx,null!=d&&((a=document.createElement("i")).tabIndex=0,a.className="bi-megaphone-fill",a.style="color: cornflowerblue;",a.role="button",a.ariaLabel="dx_operations",a.setAttribute("data-bs-container","body"),a.setAttribute("data-bs-toggle","popover"),a.setAttribute("data-bs-trigger","focus"),a.setAttribute("data-bs-sanitizer","true"),a.setAttribute("data-bs-placement","auto"),a.setAttribute("data-bs-html","true"),a.setAttribute("data-bs-title","Announced DX Op.: "+d.summary),a.setAttribute("data-bs-content",d.description+"data from "+t),s.appendChild(a)),document.createElement("td"));o.appendChild(l),c.appendChild(o),c.append(s),r.appendChild(c);try{var i=document.createElement("span"),m=(i.className="img-flag fi fi-"+e.iso,i.setAttribute("data-bs-container","body"),i.setAttribute("data-bs-toggle","popover"),i.setAttribute("data-bs-trigger","hover"),i.setAttribute("data-bs-placement","left"),i.setAttribute("data-bs-content",e.country),document.createElement("td"));m.appendChild(i),r.appendChild(m)}catch(e){console.log(e),console.log("error creating flag");d=document.createElement("td");r.appendChild(d)}t=document.createElement("td"),t.className="d-none d-lg-table-cell d-xl-table-cell",t.textContent=e.country,r.appendChild(t),a=document.createElement("td");a.className="d-none d-lg-table-cell d-xl-table-cell";try{a.textContent=e.comm.substring(0,100)}catch(e){a.textContent=""}r.appendChild(a);let u=new Date(1e3*e.time),p="00"+u.getUTCHours(),h=(p=p.substring(p.length-2,p.length),"00"+u.getMinutes()),b=(h=h.substring(h.length-2,h.length),"00"+u.getUTCDate()),g=(b=b.substring(b.length-2,b.length),"00"+(Number(u.getUTCMonth())+1));g=g.substring(g.length-2,g.length);l=u.getUTCFullYear(),o=p+":"+h,u=b+"/"+g+"/"+l,s=document.createElement("div"),s.className="d-flex flex-column",c=document.createElement("div");return c.textContent=o,s.appendChild(c),u!=n&&((i=document.createElement("div")).textContent=u,s.appendChild(i)),r.appendChild(s),r}build(n,a){if(null!=n){var r=new Date;let e="00"+r.getUTCDate(),t=(e=e.substring(e.length-2,e.length),"00"+(Number(r.getUTCMonth())+1));t=t.substring(t.length-2,t.length);var r=r.getUTCFullYear(),l=e+"/"+t+"/"+r,o=(document.getElementById(this.selector).replaceChildren(),[]);for(let e=0;ee.value)).lengthe.json()).then(t=>{try{tb.build(t)}catch(e){console.log(e),console.log(e.stack),console.log(t)}})} +class table_builder{constructor(e){this.selector=e,this.current_data=[],this.first_time=!0}getLastRowId(){let e;return e=null==this.current_data||this.current_data.length<1?0:this.current_data[0].rowid}resetData(){this.current_data=[]}#buildRow(e,t,n,a=""){var r=document.createElement("tr"),t=(0NG3K Website",l=document.createElement("i"),o=(l.className="bi-search",l.role="button",l.ariaLabel=e.dx,document.createElement("a")),s=(o.href=qrz_url+e.dx,o.target="_blank",o.rel="noopener",document.createElement("span")),c=document.createElement("mark"),c=(c.textContent=e.dx,e.dx==a?s.appendChild(c):s.textContent=" "+e.dx,null!=d&&((a=document.createElement("i")).tabIndex=0,a.className="bi-megaphone-fill",a.style="color: cornflowerblue;",a.role="button",a.ariaLabel="dx_operations",a.setAttribute("data-bs-container","body"),a.setAttribute("data-bs-toggle","popover"),a.setAttribute("data-bs-trigger","focus"),a.setAttribute("data-bs-sanitizer","true"),a.setAttribute("data-bs-placement","auto"),a.setAttribute("data-bs-html","true"),a.setAttribute("data-bs-title","Announced DX Op.: "+d.summary),a.setAttribute("data-bs-content",d.description+"data from "+t),s.appendChild(a)),document.createElement("td"));o.appendChild(l),c.appendChild(o),c.append(s),r.appendChild(c);try{var i=document.createElement("span"),m=(i.className="img-flag fi fi-"+e.iso,i.setAttribute("data-bs-container","body"),i.setAttribute("data-bs-toggle","popover"),i.setAttribute("data-bs-trigger","hover"),i.setAttribute("data-bs-placement","left"),i.setAttribute("data-bs-content",e.country),document.createElement("td"));m.appendChild(i),r.appendChild(m)}catch(e){console.log(e),console.log("error creating flag");d=document.createElement("td");r.appendChild(d)}t=document.createElement("td"),t.className="d-none d-lg-table-cell d-xl-table-cell",t.textContent=e.country,r.appendChild(t),a=document.createElement("td");a.className="d-none d-lg-table-cell d-xl-table-cell";try{a.textContent=e.comm.substring(0,100)}catch(e){a.textContent=""}r.appendChild(a);let u=new Date(1e3*e.time),p="00"+u.getUTCHours(),h=(p=p.substring(p.length-2,p.length),"00"+u.getMinutes()),b=(h=h.substring(h.length-2,h.length),"00"+u.getUTCDate()),g=(b=b.substring(b.length-2,b.length),"00"+(Number(u.getUTCMonth())+1));g=g.substring(g.length-2,g.length);l=u.getUTCFullYear(),o=p+":"+h,u=b+"/"+g+"/"+l,s=document.createElement("div"),s.className="d-flex flex-column",c=document.createElement("div");return c.textContent=o,s.appendChild(c),u!=n&&((i=document.createElement("div")).textContent=u,s.appendChild(i)),r.appendChild(s),r}build(n,a){if(null!=n){var r=new Date;let e="00"+r.getUTCDate(),t=(e=e.substring(e.length-2,e.length),"00"+(Number(r.getUTCMonth())+1));t=t.substring(t.length-2,t.length);var r=r.getUTCFullYear(),l=e+"/"+t+"/"+r,o=(document.getElementById(this.selector).replaceChildren(),[]);for(let e=0;ee.value)).lengthe.json()).then(t=>{try{tb.build(t)}catch(e){console.log(e),console.log(e.stack),console.log(t)}})} diff --git a/webapp.py b/webapp.py index 33cb05a..40fd75b 100644 --- a/webapp.py +++ b/webapp.py @@ -382,7 +382,7 @@ def find_callsign(): def get_heatmap_data(): #continent = request.args.get("continent") continent = request.json['continent'] - logger.debug(request.get_json()); + logger.debug(request.get_json()) response = flask.Response(json.dumps(heatmap_cbp.get_data(continent))) logger.debug(response) if response is None: