20201107

2024-09-21 07:27:09 +00:00 · 2020-11-07 10:03:40 +00:00 · 2020-11-07 10:03:40 +00:00 · 4205522132
commit 4205522132
parent 4dff6b278b
14 changed files with 238 additions and 37 deletions
--- a/README.md
+++ b/README.md
@ -211,6 +211,9 @@ When you are on-line, if you would to index your website on search engines, you
 **Index on MySQL
 If you would to increase speed on callsign search, you could define some index on the table 'spot'. You can see more details on 'scripts/create_mysql_index.sql'
 ~  
+
+### Monitoring
+you can use the scritp "scripts/monitoring.sh" in order to monitoring your sistem. Check instruction inside this scripts.
 ### Screenshots
 ----------
 <img src="docs/images/01_desktop_main.jpg" width="300"/>
--- a/cfg/config.json.template
+++ b/cfg/config.json.template
@ -19,6 +19,7 @@
            {"label":"Plots", "link": "/plots.html", "external": false},
            {"label":"Sources", "link": "https://github.com/coulisse/spiderweb/" , "external": true},    
            {"label":"Cookies policy", "link": "/cookies.html", "external": false}
+            {"label":"Privacy policy", "link": "../static/html/privacy.html", "external": false}
            ]
        }
 }
--- a/cfg/log_config.ini
+++ b/cfg/log_config.ini
@ -8,18 +8,18 @@ keys=stream_handler,file_handler
 keys=formatter

 [logger_root]
-level=INFO  
+level=DEBUG 
 handlers=stream_handler,file_handler

 [handler_stream_handler]
 class=StreamHandler
-level=INFO 
+level=DEBUG
 formatter=formatter
 args=(sys.stderr,)

 [handler_file_handler]
 class=FileHandler
-level=INFO 
+level=DEBUG
 formatter=formatter
 args=("logs", "w")

--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@ -10,6 +10,9 @@ xx/xx/2020: 2.0
 - removed link to country-flags.io and used https://github.com/lipis/flag-icon-css hosted at https://cdnjs.com/libraries/flag-icon-css/3.4.1 in order to improve performances and remove cookies
 - removed cookie bar
 - improved offline page
+- remove colors from callsing search resoults table
+- added popover on mobile
+- added monitor.sh in order to monitor your system
 ___
 04/10/2020: 1.2.2
 - Added config.sh utilty in script folder, in order to avoid edit manually config.json
--- a/docs/TODO.md
+++ b/docs/TODO.md
@ -1,5 +1,23 @@

 ### TODO
+- [ ] migrate to boostrap 5 (when ready.. end of 2020?)
+  - [x] fix frequency badge
+  - [x] fix menu
+  - [x] fix filter
+  - [x] fix heatmap on plots
+  - [x] check progressive webapp
+  - [x] check jumbotron margin / padding
+  - [x] check owasp security
+  - [x] check ssl
+  - [x] add padding to menu in mobile version
+  - [x] space between filter and table
+  - [x] fix black row at table start
+  - [x] remove transparent background filter on mobile
+  - [x] reduce jumbotron on mobile
+  - [x] fix margin on cookies
+  - [ ] use bootstrap 5 icons
+  - [ ] test bootstrap 5 beta
+  - [ ] test bootstrap 5 stable release
 - [x] add comments to code
 - [ ] add other plots (cpu load, memory used, disk used)
 - [ ] manage common html parts with "includes" https://jinja.palletsprojects.com/en/2.11.x/templates/#template-inheritance
@ -14,30 +32,21 @@
 - [x] add configuration to menu
 - [x] edit configuration with cli interface
 - [x] migrate to python 3  
- [ ] migrate to boostrap 5 (when ready.. end of 2020?)
-  - [x] fix frequency badge
-  - [x] fix menu
-  - [x] fix filter
-  - [x] fix heatmap on plots
-  - [ ] check progressive webapp
-  - [x] check jumbotron margin / padding
-  - [ ] check owasp security
-  - [x] check ssl
-  - [ ] caching countries json
-  - [x] add padding to menu in mobile version
-  - [x] space between filter and table
-  - [x] fix black row at table start
-  - [x] remove transparent background filter on mobile
-  - [x] reduce jumbotron on mobile
-  - [x] fix margin on cookies
-  - [ ] use bootstrap 5 icons
 - [x] set default icon image for images not found (es for 4U1UN)
 - [ ] implement CSRF protection. See:   https://flask-wtf.readthedocs.io/en/stable/csrf.html
- [ ] manage country tooltip on mobile (maybe after bootstrap 5)
+- [x] manage country tooltip on mobile (maybe after bootstrap 5)
 - [x] add ssb/cw/digi filter (required by IU5HES) 
 - [x] on search collapse filter
 - [x] improve offline page
 - [x] add telnet address and contatc mail to config.json (edit also the menu utility)
 - [x] fix shell (.sh) scripts for launch plots
 - [x] update README with cloudflare SSL
- [ ] try it on bjorne
+- [x] try it on bjorne
+- [ ] caching countries json
+- [x] change search callsign table (remove colors)
+- [ ] put all images on a CDN
+- [ ] define a new icon (favicon + progressive webapp icon)
+- [x] add script for send monitoring mail
+- [x] fix month plots
+- [ ] add native cookie bar
+- [ ] privacy policy
--- a/scripts/ciao.txt
+++ b/scripts/ciao.txt
@ -0,0 +1 @@
+hellohellohello
--- a/scripts/monitor.sh
+++ b/scripts/monitor.sh
@ -0,0 +1,114 @@
+#!/bin/sh
+#-------------------------------------------------------------------------
+# Author: IU1BOW - Corrado Gerbaldo
+#.........................................................................
+# Script for monitoring dxspider system
+#
+# To use this script you need to install and configure ssmtp
+# for enable sending mail using gmail:
+# 1) Login to your gmail account.
+# 2) Go to https://www.google.com/settings/security/lesssecureapps and Turn On this feature.
+# 3) Go to https://accounts.google.com/DisplayUnlockCaptcha and click Continue.
+#-------------------------------------------------------------------------
+DISK=/dev/sda1       # <--- CHANGE WITH YOUR DRIVE !!!
+
+#...................................................
+CONFIG=../cfg/config.json
+LIM_DISK=80
+LIM_MEMPERC=80
+LIM_DATE=1800
+
+DIR=`realpath -s $0|sed 's|\(.*\)/.*|\1|'`
+echo Absolute path: ${DIR}
+cd ${DIR}
+
+WARNING=false
+trap 'rm -f "$TMPFILE"' EXIT
+TMPFILE=$(mktemp)|| exit 1
+
+#echo 'Subject: This is dxspider monitor '>>${TMPFILE}
+
+echo >> ${TMPFILE}
+echo 'RAM:' >> ${TMPFILE}
+mon_memperc=`free | grep Mem | awk '{print $3/$2 * 100}'`
+mon_memperc=`echo ${mon_memperc}| awk '{ printf "%d\n",$1 }'`
+free -h>>${TMPFILE}
+if [ ${mon_memperc} -gt ${LIM_MEMPERC} ]
+then
+	WARNING=true
+	echo "WARNING: RAM space is critical!">> ${TMPFILE}
+fi
+echo  >> ${TMPFILE}
+
+echo 'DISK' >> ${TMPFILE}
+mon_diskperc=`df ${DISK} | tail -n 1 | grep -E [[:digit:]]+% -o | grep -E [1-9]+ -o`
+df ${DISK} -h>>${TMPFILE}
+if [ ${mon_diskperc} -gt ${LIM_DISK} ]
+then
+	WARNING=true
+	echo "WARNING: Disk space is critical!">> ${TMPFILE}
+fi
+echo  >> ${TMPFILE}
+
+echo 'SERVICES' >> ${TMPFILE}
+ps -ef|head -1>> ${TMPFILE}
+ps -ef | grep cluster.pl | grep -v grep >> ${TMPFILE}
+mon_cluster=$?
+ps -ef | grep mysqld | grep -v grep >> ${TMPFILE}
+mon_mariadb=$?
+ps -ef | grep "wsgi.py" | grep -v grep >> ${TMPFILE}
+mon_wsgi=$?
+ps -ef | grep "nginx: worker" | grep -v grep >> ${TMPFILE}
+mon_nginx=$?
+
+echo  >> ${TMPFILE}
+
+if [ ${mon_mariadb} -ne 0 ]  
+then
+	WARNING=true
+	echo "ERROR: maria db is not running!">> ${TMPFILE}
+else 
+	echo 'Mysql dxspider' >> ${TMPFILE}
+	password=`grep -Po '"passwd":.*?[^\/]",' ${CONFIG}|cut -d '"' -f 4`
+	user=`grep -Po '"user":.*?[^\/]",' ${CONFIG}|cut -d '"' -f 4`
+	mon_sqlres=`mysql --user=$user --password=$password -Bse "use dxcluster;select time from spot order by 1 desc limit 1;"`
+
+	current_date=`date +"%s"`
+	date_diff=$((current_date - mon_sqlres))
+	echo 'Last spot received: ' ${date_diff}' seconds ago' >> ${TMPFILE}
+	if [ ${date_diff} -gt ${LIM_DATE} ]
+	then
+		WARNING=true
+		echo 'WARNING: mysql is not UPDATED!' >> ${TMPFILE}
+	fi
+fi
+
+if [ ${mon_cluster} -ne 0 ]  
+then
+	WARNING=true
+	echo "ERROR: cluster is not running!">> ${TMPFILE}
+fi
+
+if [ ${mon_wsgi} -ne 0 ]  
+then
+	WARNING=true
+	echo "ERROR: WSGI is not running!">> ${TMPFILE}
+fi
+
+if [ ${mon_nginx} -ne 0 ]  
+then
+	WARNING=true
+	echo "ERROR: NGINX is not running!">> ${TMPFILE}
+fi
+
+if [ ${WARNING} = true ] ; then
+	echo "$(echo 'Subject: [WARNING]: Spider monitor'; cat ${TMPFILE})" > ${TMPFILE}
+else   
+	echo "$(echo 'Subject: Spider monitor'; cat ${TMPFILE})" > ${TMPFILE}
+fi
+
+mailto=`grep -Po '"mail":.*?[^\/]",' ${CONFIG}|cut -d '"' -f 4`
+ssmtp ${mailto} < ${TMPFILE}
+#TODO: add logic in order to send mail only in case of fail (ora one mail per day)                                                    
+cat ${TMPFILE}
+rm ${TMPFILE}
--- a/scripts/qso_months.py
+++ b/scripts/qso_months.py
@ -12,6 +12,7 @@ import logging.config
 import matplotlib.gridspec as gridspec
 from qry import *
 from plotuty import saveplt
+from calendar import monthrange

 logging.config.fileConfig("../cfg/log_config.ini", disable_existing_loggers=False)
 file_output = '../static/plots/'+ os.path.splitext(os.path.basename(sys.argv[0]))[0]
@ -24,10 +25,12 @@ qry_string="""

 	select 
 		concat(YEAR(FROM_UNIXTIME(time)),'-',  
-		MONTH(FROM_UNIXTIME(time))) as ym, 
+		right(concat('0',MONTH(FROM_UNIXTIME(time))),2)
+                ) as ym, 
 		count(0) as number
 		from spot 
-		GROUP by 1;
+		GROUP by 1
+                ORDER by 1;

    """
 logging.debug(qry_string) 
@ -56,6 +59,15 @@ plt.grid(False)
 plt.subplots_adjust(left=0.15)

 plt.bar(x, y, align='center')
+list_y = list(y)
+le=len(list_y)-1
+day=datetime.today().day
+month=datetime.today().month
+year=datetime.today().year
+days_of_month=monthrange(year,month)
+list_y[le]=int(y[le]/day*days_of_month[1])
+y = tuple(list_y)
+plt.scatter(x,y)

 saveplt(plt,file_output)

--- a/static/country.json
+++ b/static/country.json
--- a/static/css/style.css
+++ b/static/css/style.css
@ -113,4 +113,3 @@ span.search-callsign {
 #band {
 	margin-top: 5px;
 }
-
--- a/static/html/privacy.html
+++ b/static/html/privacy.html
@ -0,0 +1,51 @@
+<h1>Privacy Policy for this web site</h1>
+<p>At this web site, one of our main priorities is the privacy of our visitors. This Privacy Policy document contains types of information that is collected and recorded by this web site and how we use it.</p>
+<p>If you have additional questions or require more information about our Privacy Policy, do not hesitate to contact us.</p>
+<p>This Privacy Policy applies only to our online activities and is valid for visitors to our website with regards to the information that they shared and/or collect in this web site. This policy is not applicable to any information collected offline or via channels other than this website. Our Privacy Policy was created with the help of the <a href="https://www.privacypolicygenerator.info">Privacy Policy Generator</a> and the <a href="https://www.generateprivacypolicy.com/">Free Privacy Policy Generator</a>.</p>
+<h2>Consent</h2>
+<p>By using our website, you hereby consent to our Privacy Policy and agree to its terms. For our Terms and Conditions, please visit the <a href="https://www.privacypolicyonline.com/terms-conditions-generator/">Terms & Conditions Generator</a>.</p>
+<h2>Information we collect</h2>
+<p>The personal information that you are asked to provide, and the reasons why you are asked to provide it, will be made clear to you at the point we ask you to provide your personal information.</p>
+<p>If you contact us directly, we may receive additional information about you such as your name, email address, phone number, the contents of the message and/or attachments you may send us, and any other information you may choose to provide.</p>
+<p>When you register for an Account, we may ask for your contact information, including items such as name, company name, address, email address, and telephone number.</p>
+<h2>How we use your information</h2>
+<p>We use the information we collect in various ways, including to:</p>
+<ul>
+<li>Provide, operate, and maintain our webste</li>
+<li>Improve, personalize, and expand our webste</li>
+<li>Understand and analyze how you use our webste</li>
+<li>Develop new products, services, features, and functionality</li>
+<li>Communicate with you, either directly or through one of our partners, including for customer service, to provide you with updates and other information relating to the webste, and for marketing and promotional purposes</li>
+<li>Send you emails</li>
+<li>Find and prevent fraud</li>
+</ul>
+<h2>Log Files</h2>
+<p>this web site follows a standard procedure of using log files. These files log visitors when they visit websites. All hosting companies do this and a part of hosting services' analytics. The information collected by log files include internet protocol (IP) addresses, browser type, Internet Service Provider (ISP), date and time stamp, referring/exit pages, and possibly the number of clicks. These are not linked to any information that is personally identifiable. The purpose of the information is for analyzing trends, administering the site, tracking users' movement on the website, and gathering demographic information.</p>
+<h2>Cookies and Web Beacons</h2>
+<p>Like any other website, this web site uses 'cookies'. These cookies are used to store information including visitors' preferences, and the pages on the website that the visitor accessed or visited. The information is used to optimize the users' experience by customizing our web page content based on visitors' browser type and/or other information.</p>
+<p>For more general information on cookies, please read <a href="https://www.privacypolicies.com/blog/cookies/">"What Are Cookies"</a>.</p>
+<h2>Advertising Partners Privacy Policies</h2>
+<P>You may consult this list to find the Privacy Policy for each of the advertising partners of this web site.</p>
+<p>Third-party ad servers or ad networks uses technologies like cookies, JavaScript, or Web Beacons that are used in their respective advertisements and links that appear on this web site, which are sent directly to users' browser. They automatically receive your IP address when this occurs. These technologies are used to measure the effectiveness of their advertising campaigns and/or to personalize the advertising content that you see on websites that you visit.</p>
+<p>Note that this web site has no access to or control over these cookies that are used by third-party advertisers.</p>
+<h2>Third Party Privacy Policies</h2>
+<p>this web site's Privacy Policy does not apply to other advertisers or websites. Thus, we are advising you to consult the respective Privacy Policies of these third-party ad servers for more detailed information. It may include their practices and instructions about how to opt-out of certain options. </p>
+<p>You can choose to disable cookies through your individual browser options. To know more detailed information about cookie management with specific web browsers, it can be found at the browsers' respective websites.</p>
+<h2>CCPA Privacy Rights (Do Not Sell My Personal Information)</h2>
+<p>Under the CCPA, among other rights, California consumers have the right to:</p>
+<p>Request that a business that collects a consumer's personal data disclose the categories and specific pieces of personal data that a business has collected about consumers.</p>
+<p>Request that a business delete any personal data about the consumer that a business has collected.</p>
+<p>Request that a business that sells a consumer's personal data, not sell the consumer's personal data.</p>
+<p>If you make a request, we have one month to respond to you. If you would like to exercise any of these rights, please contact us.</p>
+<h2>GDPR Data Protection Rights</h2>
+<p>We would like to make sure you are fully aware of all of your data protection rights. Every user is entitled to the following:</p>
+<p>The right to access – You have the right to request copies of your personal data. We may charge you a small fee for this service.</p>
+<p>The right to rectification – You have the right to request that we correct any information you believe is inaccurate. You also have the right to request that we complete the information you believe is incomplete.</p>
+<p>The right to erasure – You have the right to request that we erase your personal data, under certain conditions.</p>
+<p>The right to restrict processing – You have the right to request that we restrict the processing of your personal data, under certain conditions.</p>
+<p>The right to object to processing – You have the right to object to our processing of your personal data, under certain conditions.</p>
+<p>The right to data portability – You have the right to request that we transfer the data that we have collected to another organization, or directly to you, under certain conditions.</p>
+<p>If you make a request, we have one month to respond to you. If you would like to exercise any of these rights, please contact us.</p>
+<h2>Children's Information</h2>
+<p>Another part of our priority is adding protection for children while using the internet. We encourage parents and guardians to observe, participate in, and/or monitor and guide their online activity.</p>
+<p>this web site does not knowingly collect any Personal Identifiable Information from children under the age of 13. If you think that your child provided this kind of information on our website, we strongly encourage you to contact us immediately and we will do our best efforts to promptly remove such information from our records.</p>
--- a/static/js/application.js
+++ b/static/js/application.js
@ -27,7 +27,9 @@ function findCountry(countries, wpx_to_find) {
 * @param callsign {string} An optional parameter with the callsign to search
 */
 function buildHtmlTable(selector,data,rl,countries,callsign) {
-	//rows
+  if  ( data != null ){
+
+	//	$('[data-toggle="popover"]').popover('dispose');

 	var myRows=new Array();

@ -47,12 +49,11 @@ function buildHtmlTable(selector,data,rl,countries,callsign) {
 		var found = rl.find(element => element ==data[i].rowid);
 		if ( callsign != undefined ) {
 			if ( callsign == data[i].de ) {
-				row$=$('<tr class="table-success" id="'+data[i].rowid+'"/>');
+				row$=$('<tr id="'+data[i].rowid+'"/>');
 			} else if ( callsign == data[i].dx ) {
-				row$=$('<tr class="table-info" id="'+data[i].rowid+'"/>');
+				row$=$('<tr id="'+data[i].rowid+'"/>');
 			}
 		} else if ( found == undefined && rl.length > 0) {
-//		if ( found == undefined && rl.length > 0) {
 			row$=$('<tr class="table-info" id="'+data[i].rowid+'"/>');
 		};

@ -73,11 +74,10 @@ function buildHtmlTable(selector,data,rl,countries,callsign) {
 			dx = data[i].dx
 		};

-
 		row$.append($('<td/>').html('<a href="https://www.qrz.com/db/'+data[i].dx+ '" target="_blank" rel="noopener"><i class="search" aria-label="'+data[i].dx+'"></i></a><span>&nbsp'+dx+'</span>'));
 		try {
-  		//	row$.append($('<td/>').html('<a href="#" data-toggle="tooltip" title="'+country.country+'"><img class="img-flag" src="https://www.countryflags.io/'+country.ISO+'/shiny/32.png" alt="'+country.country+'"></a>'));
-  			row$.append($('<td/>').html('<span class="img-flag flag-icon flag-icon-'+country.ISO.toLowerCase()+'"></span>'));
+			row$.append($('<td/>').html('<span class="img-flag flag-icon flag-icon-'+country.ISO+'" data-container="body" data-toggle="popover" data-trigger="hover" data-placement="left" data-content="'+country.country+'"></span>'));
+//			row$.append($('<td/>').html('<span class="img-flag flag-icon flag-icon-'+country.ISO+'" data-toggle="tooltip" data-trigger="hover" title="'+country.country+'"></span>'));
 		} catch (err) {
 			row$.append($('<td/>'));
 		};
@ -105,11 +105,18 @@ function buildHtmlTable(selector,data,rl,countries,callsign) {
    		$(selector).append(row$);
   	}

+	$(function () {
+  		$('[data-toggle="popover"]').popover({
+    			container: selector
+  		})
+	})
+
 	try {
 		return Array.from(myRows);
 	} catch (err) {
 		return;
 	}
+    }
 };

 /*
@ -194,7 +201,8 @@ function myTimer() {
 		try {
 			rows_list=buildHtmlTable('#bodyspot',JSON.parse(this.response),rows_list,my_countries.country_codes );
 		} catch (err) {
-			console.log(this.response);
+			console.log(err);
+			console.log(err.stack);
 		}
 	}
 	request.send()
@ -240,7 +248,7 @@ function plotsTimer() {
 			plot_list = buildHtmlPlots('#plotlist',JSON.parse(this.response));
 		} catch (err) {
 			console.log(err);
-			console.log(this.response);
+			console.log(err.stack);
 		}
 	}
 	request.send()
--- a/static/js/min/application.min.js
+++ b/static/js/min/application.min.js
--- a/templates/index.html
+++ b/templates/index.html
@ -12,7 +12,7 @@
 	<link rel="apple-touch-icon" href="/static/images/icons/icon-apple.png">
 	<link rel="stylesheet" href="/static/css/min/style.min.css"> 

-	<link rel="preload" href="https://stackpath.bootstrapcdn.com/bootstrap/5.0.0-alpha2/css/bootstrap.min.css" as="style" integrity="sha384-DhY6onE6f3zzKbjUPRc2hOzGAdEf4/Dz+WJwBvEYL/lkkIsI3ihufq9hk9K4lVoK" crossorigin="anonymous" onload="this.rel='stylesheet'">
+	<link rel="preload" href="https://stackpath.bootstrapcdn.com/bootstrap/5.0.0-alpha2/css/bootstrap.min.css" as="style" integrity="sha384-DhY6onE6f3zzKbjUPRc2hOzGAdEf4/Dz+WJwBvEYL/lkkIsI3ihufq9hk9K4lVoK" crossorigin="anonymous" onload="this.rel='stylesheet'" media="all">
  	<noscript><link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/5.0.0-alpha2/css/bootstrap.min.css"></noscript>

 	<link rel="preload" href="https://cdnjs.cloudflare.com/ajax/libs/flag-icon-css/3.4.1/css/flag-icon.min.css" as="style" integrity="sha512-RYqXH3Tn/sC/iuKrQ0idMEQFSU7HEuPpQDhd/kzsIkE/1WxG43RN2RGp9bx2sX/Gdh1G0Gn3zActuCHqE4t6rg==" crossorigin="anonymous" onload="this.rel='stylesheet'">