hamradio/spiderweb
1
0
Fork 0
mirror of https://github.com/coulisse/spiderweb.git synced 2024-09-21 07:27:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'development'

Browse Source
This commit is contained in:
coulisse 2024-03-10 14:39:47 +01:00
commit 2102a58251
20 changed files with 932 additions and 2220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CITATION.cff
View File

@ -24,5 +24,5 @@ keywords:
- dxcluster
- spiderweb
license: GPL-3.0
version: v2.5.2
date-released: 2023-12-02
version: v2.5.3
date-released: 2024-03-10

2
README.md
View File

@ -9,7 +9,7 @@
[![CodeFactor](https://www.codefactor.io/repository/github/coulisse/spiderweb/badge)](https://www.codefactor.io/repository/github/coulisse/spiderweb)
- **Release:** v2.5.2
- **Release:** v2.5.3
- **Author:** Corrado Gerbaldo - [IU1BOW](https://www.qrz.com/db/IU1BOW)
- **Mail:** <corrado.gerbaldo@gmail.com>
- **Licensing:** Gpl V3.0 see [LICENSE](LICENSE) file.

1
cfg/config.json.template
View File

@ -10,7 +10,6 @@
},
"mycallsign":"XXXXXX",
"mail":"foo@bar.com",
"mail_token": "foobar",
"enable_cq_filter":"n",
"telnet": {
"host": "mysite",

2
cfg/version.txt
View File

@ -1 +1 @@
v2.5.2
v2.5.3

12
docs/CHANGELOG.md
View File

@ -1,5 +1,14 @@
### Change log
Date: 02/12/2023
Date: 10/03/2024
Release: v2.5.3
- adapted card size and text for mobile
- removed monitor
- removed cookie consent banner, since this application uses only technical cookies
- issue [#51] (https://github.com/coulisse/spiderweb/issues/51) -- just for caching
- security [#22] (https://github.com/coulisse/spiderweb/security/dependabot/22)
___
Date: 03/12/2023
Release: v2.5.2
- security issue #46.
- csp report
@ -8,7 +17,6 @@ Release: v2.5.2
- Sanitized callsign input
- Added propagation page with MUF Map. Issue [#27](https://github.com/coulisse/spiderweb/issues/27). Thanks to Paul Herman and Andrew Rodland
___
Date: 12/11/2023
Release: v2.4.5.1

191
lib/qry_builder.py Normal file
View File

@ -0,0 +1,191 @@
# find id in json : ie frequency / continent
def find_id_json(json_object, name):
return [obj for obj in json_object if obj["id"] == name][0]
def query_build_callsign(logger,callsign):
query_string = ""
if len(callsign) <= 14:
query_string = (
"(SELECT rowid, spotter AS de, freq, spotcall AS dx, comment AS comm, time, spotdxcc from spot WHERE spotter='"
+ callsign
+ "'"
)
query_string += " ORDER BY rowid desc limit 10)"
query_string += " UNION "
query_string += (
"(SELECT rowid, spotter AS de, freq, spotcall AS dx, comment AS comm, time, spotdxcc from spot WHERE spotcall='"
+ callsign
+ "'"
)
query_string += " ORDER BY rowid desc limit 10);"
else:
logger.warning("callsign too long")
return query_string
def query_build(logger,parameters,band_frequencies,modes_frequencies,continents_cq,enable_cq_filter):
try:
last_rowid = str(parameters["lr"]) # Last rowid fetched by front end
get_param = lambda parameters, parm_name: parameters[parm_name] if (parm_name in parameters) else []
dxcalls=get_param(parameters, "dxcalls")
band=get_param(parameters, "band")
dere=get_param(parameters, "de_re")
dxre=get_param(parameters, "dx_re")
mode=get_param(parameters, "mode")
exclft8=get_param(parameters, "exclft8")
exclft4=get_param(parameters, "exclft4")
decq = []
if "cqdeInput" in parameters:
decq[0] = parameters["cqdeInput"]
dxcq = []
if "cqdxInput" in parameters:
dxcq[0] = parameters["cqdxInput"]
query_string = ""
#construct callsign of spot dx callsign
dxcalls_qry_string = " AND spotcall IN (" + ''.join(map(lambda x: "'" + x + "'," if x != dxcalls[-1] else "'" + x + "'", dxcalls)) + ")"
# construct band query decoding frequencies with json file
band_qry_string = " AND (("
for i, item_band in enumerate(band):
freq = find_id_json(band_frequencies["bands"], item_band)
if i > 0:
band_qry_string += ") OR ("
band_qry_string += (
"freq BETWEEN " + str(freq["min"]) + " AND " + str(freq["max"])
)
band_qry_string += "))"
# construct mode query
mode_qry_string = " AND (("
for i, item_mode in enumerate(mode):
single_mode = find_id_json(modes_frequencies["modes"], item_mode)
if i > 0:
mode_qry_string += ") OR ("
for j in range(len(single_mode["freq"])):
if j > 0:
mode_qry_string += ") OR ("
mode_qry_string += (
"freq BETWEEN "
+ str(single_mode["freq"][j]["min"])
+ " AND "
+ str(single_mode["freq"][j]["max"])
)
mode_qry_string += "))"
#Exluding FT8 or FT4 connection
ft8_qry_string = " AND ("
if exclft8:
ft8_qry_string += "(comment NOT LIKE '%FT8%')"
single_mode = find_id_json(modes_frequencies["modes"], "digi-ft8")
for j in range(len(single_mode["freq"])):
ft8_qry_string += (
" AND (freq NOT BETWEEN "
+ str(single_mode["freq"][j]["min"])
+ " AND "
+ str(single_mode["freq"][j]["max"])
+ ")"
)
ft8_qry_string += ")"
ft4_qry_string = " AND ("
if exclft4:
ft4_qry_string += "(comment NOT LIKE '%FT4%')"
single_mode = find_id_json(modes_frequencies["modes"], "digi-ft4")
for j in range(len(single_mode["freq"])):
ft4_qry_string += (
" AND (freq NOT BETWEEN "
+ str(single_mode["freq"][j]["min"])
+ " AND "
+ str(single_mode["freq"][j]["max"])
+ ")"
)
ft4_qry_string += ")"
# construct DE continent region query
dere_qry_string = " AND spottercq IN ("
for i, item_dere in enumerate(dere):
continent = find_id_json(continents_cq["continents"], item_dere)
if i > 0:
dere_qry_string += ","
dere_qry_string += str(continent["cq"])
dere_qry_string += ")"
# construct DX continent region query
dxre_qry_string = " AND spotcq IN ("
for i, item_dxre in enumerate(dxre):
continent = find_id_json(continents_cq["continents"], item_dxre)
if i > 0:
dxre_qry_string += ","
dxre_qry_string += str(continent["cq"])
dxre_qry_string += ")"
if enable_cq_filter == "Y":
# construct de cq query
decq_qry_string = ""
if len(decq) == 1:
if decq[0].isnumeric():
decq_qry_string = " AND spottercq =" + decq[0]
# construct dx cq query
dxcq_qry_string = ""
if len(dxcq) == 1:
if dxcq[0].isnumeric():
dxcq_qry_string = " AND spotcq =" + dxcq[0]
if last_rowid is None:
last_rowid = "0"
if not last_rowid.isnumeric():
last_rowid = 0
query_string = "SELECT rowid, spotter AS de, freq, spotcall AS dx, comment AS comm, time, spotdxcc from spot WHERE rowid > "+(str(last_rowid))
if dxcalls:
query_string += dxcalls_qry_string
if len(band) > 0:
query_string += band_qry_string
if len(mode) > 0:
query_string += mode_qry_string
if exclft8:
query_string += ft8_qry_string
if exclft4:
query_string += ft4_qry_string
if len(dere) > 0:
query_string += dere_qry_string
if len(dxre) > 0:
query_string += dxre_qry_string
if enable_cq_filter == "Y":
if len(decq_qry_string) > 0:
query_string += decq_qry_string
if len(dxcq_qry_string) > 0:
query_string += dxcq_qry_string
query_string += " ORDER BY rowid desc limit 50;"
logger.debug (query_string)
except Exception as e:
logger.error(e)
query_string = ""
return query_string
def query_build_callsing_list():
query_string = "SELECT spotcall AS dx FROM (select spotcall from spot order by rowid desc limit 50000) s1 GROUP BY spotcall ORDER BY count(spotcall) DESC, spotcall LIMIT 100;"
return query_string

44
requirements.txt
View File

@ -1,29 +1,41 @@
blinker==1.7.0
charset-normalizer==3.3.2
click==8.1.7
Flask==3.0.0
Flask-Minify==0.42
Flask-WTF==1.2.1
astroid==2.12.14
blinker==1.6.2
charset-normalizer==2.1.1
click==8.1.3
dill==0.3.6
docopt-ng==0.8.1
easywatch==0.0.5
Flask==2.3.3
Flask-Consent==0.0.3
Flask-Minify==0.41
Flask-WTF==1.1.1
htmlmin==0.1.12
idna==3.4
isort==5.11.4
itsdangerous==2.1.2
Jinja2==3.1.2
Jinja2==3.1.3
jsmin==3.0.1
lazy-object-proxy==1.9.0
lesscpy==0.15.1
MarkupSafe==2.1.3
markup==0.2
MarkupSafe==2.1.1
mccabe==0.7.0
mysql-connector-python>=8.2.0
numpy==1.26.1
pandas==2.1.3
numpy==1.24.1
pandas==1.5.2
platformdirs==2.6.2
ply==3.11
protobuf==4.21.12
python-dateutil==2.8.2
pytz==2023.3.post1
rcssmin==1.1.2
pytz==2022.7
rcssmin==1.1.1
requests==2.31.0
six==1.16.0
tzdata==2023.3
tomlkit==0.11.6
urllib3==2.0.7
Werkzeug==3.0.1
WTForms==3.1.1
watchdog==3.0.0
Werkzeug==2.3.8
wrapt==1.14.1
WTForms==3.0.1
xmltodict==0.13.0
xxhash==3.4.1
xxhash==3.1.0

119
scripts/monitor.sh
View File

@ -1,119 +0,0 @@
#!/bin/sh
#-------------------------------------------------------------------------
# Author: IU1BOW - Corrado Gerbaldo
#.........................................................................
# Script for monitoring dxspider system
#
# To use this script you need to install and configure ssmtp
# for enable sending mail using gmail (with 2-factor autentication):
# 1. Log-in into Gmail with your account
# 2. Navigate to https://security.google.com/settings/security/apppasswords
# 3. In 'select app' choose 'custom', give it an arbitrary name and press generate
# 4. It will give you 16 chars token.
# 5. put the token in your config file
#-------------------------------------------------------------------------
DISK=/dev/sda1 # <--- CHANGE WITH YOUR DRIVE !!!
#...................................................
CONFIG=../cfg/config.json
SSMTP=/usr/sbin/ssmtp
LIM_DISK=80
LIM_MEMPERC=80
LIM_DATE=1800
DIR=$(realpath -s $0|sed 's|\(.*\)/.*|\1|')
echo Absolute path: ${DIR}
cd ${DIR}
WARNING=false
trap 'rm -f "$TMPFILE"' EXIT
TMPFILE=$(mktemp)|| exit 1
#echo 'Subject: This is dxspider monitor '>>${TMPFILE}
echo >> ${TMPFILE}
echo 'RAM:' >> ${TMPFILE}
mon_memperc=$(free | grep Mem | awk '{print $3/$2 * 100}')
mon_memperc=$(echo ${mon_memperc}| awk '{ printf "%d\n",$1 }')
free -h>>${TMPFILE}
if [ ${mon_memperc} -gt ${LIM_MEMPERC} ]
then
WARNING=true
echo "WARNING: RAM space is critical!">> ${TMPFILE}
fi
echo >> ${TMPFILE}
echo 'DISK' >> ${TMPFILE}
mon_diskperc=$(df ${DISK} | tail -n 1 | grep -E "[[:digit:]]+%" -o | grep -E "[1-9]+" -o)
df ${DISK} -h>>${TMPFILE}
if [ ${mon_diskperc} -gt ${LIM_DISK} ]
then
WARNING=true
echo "WARNING: Disk space is critical!">> ${TMPFILE}
fi
echo >> ${TMPFILE}
echo 'SERVICES' >> ${TMPFILE}
ps -ef|head -1>> ${TMPFILE}
ps -ef | grep cluster.pl | grep -v grep >> ${TMPFILE}
mon_cluster=$?
ps -ef | grep mysqld | grep -v grep >> ${TMPFILE}
mon_mariadb=$?
ps -ef | grep "wsgi.py" | grep -v grep >> ${TMPFILE}
mon_wsgi=$?
ps -ef | grep "nginx: worker" | grep -v grep >> ${TMPFILE}
mon_nginx=$?
echo >> ${TMPFILE}
if [ ${mon_mariadb} -ne 0 ]
then
WARNING=true
echo "ERROR: maria db is not running!">> ${TMPFILE}
else
echo 'Mysql dxspider' >> ${TMPFILE}
password=$(grep -Po '"passwd":.*?[^\/]",' ${CONFIG}|cut -d '"' -f 4)
user=$(grep -Po '"user":.*?[^\/]",' ${CONFIG}|cut -d '"' -f 4)
mon_sqlres=$(mysql --user=$user --password=$password -Bse "use dxcluster;select time from spot order by 1 desc limit 1;")
current_date=$(date +"%s")
date_diff=$((current_date - mon_sqlres))
echo 'Last spot received: ' ${date_diff}' seconds ago' >> ${TMPFILE}
if [ ${date_diff} -gt ${LIM_DATE} ]
then
WARNING=true
echo 'WARNING: mysql is not UPDATED!' >> ${TMPFILE}
fi
fi
if [ ${mon_cluster} -ne 0 ]
then
WARNING=true
echo "ERROR: cluster is not running!">> ${TMPFILE}
fi
if [ ${mon_wsgi} -ne 0 ]
then
WARNING=true
echo "ERROR: WSGI is not running!">> ${TMPFILE}
fi
if [ ${mon_nginx} -ne 0 ]
then
WARNING=true
echo "ERROR: NGINX is not running!">> ${TMPFILE}
fi
if [ ${WARNING} = true ] ; then
echo "$(echo 'Subject: [WARNING]: Spider monitor'; cat ${TMPFILE})" > ${TMPFILE}
else
echo "$(echo 'Subject: Spider monitor'; cat ${TMPFILE})" > ${TMPFILE}
fi
mailto=$(grep -Po '"mail":.*?[^\/]",' ${CONFIG}|cut -d '"' -f 4)
mail_token=$(grep -Po '"mail_token":.*?[^\/]",' ${CONFIG}|cut -d '"' -f 4)
echo ${mailto}
#${SSMTP} ${mailto} < ${TMPFILE}
${SSMTP} ${mailto} -au${mailto} -ap${mail_token} < ${TMPFILE}
cat ${TMPFILE}
rm ${TMPFILE}

24
static/css/dev/style.css
View File

@ -20,6 +20,7 @@
}
}
@media screen and (max-width: 768px) {
#collapseFilters.collapsing {
position: absolute !important;
@ -184,7 +185,7 @@ span.search-callsign {
.card-value {
display: block;
font-size: 200%;
font-size: 180%;
font-weight: bolder;
}
@ -194,6 +195,27 @@ span.search-callsign {
padding-left: 0.2em;
}
@media only screen and (max-width: 768px) {
.kpi-card {
width: 120px!important;
height: 120px!important;
}
.card-value {
display: block;
font-size: 150%;
font-weight: bolder;
}
.card-text {
display:block;
font-size: 80%;
padding-left: 0.2em;
}
}
#propagation-wrapper {
width: 100%;
display: flex;

2
static/css/rel/style.min.css vendored
View File

@ -1 +1 @@
@import url("https://cdn.jsdelivr.net/npm/bootstrap-icons@1.10.3/font/bootstrap-icons.css");@font-face{font-display:swap;font-family:bootstrap-icons}.badge-responsive{width:70px}@media screen and (max-width:768px){.text-responsive{font-size:12px}.badge-responsive{width:40px}#collapseFilters.collapsing{position:absolute!important;z-index:20}#collapseFilters.collapse.show{display:block;position:absolute;z-index:20}.navbar-collapse{max-height:none!important}}#form-filters{max-width:360px}.img-flag{background-color:#fff;background-size:cover!important;border:1px solid #ddd;border-radius:2px;-webkit-box-shadow:0 2px 10px rgba(0,0,0,.5),0 2px 3px rgba(0,0,0,.5);-moz-box-shadow:0 2px 10px rgba(0,0,0,.5),0 2px 3px rgba(0,0,0,.5);-o-box-shadow:0 2px 10px rgba(0,0,0,.5),0 2px 3px rgba(0,0,0,.5);box-shadow:0 2px 10px rgba(0,0,0,.5),0 2px 3px rgba(0,0,0,.5);height:19px!important;max-height:auto;max-width:auto;padding:3px;width:32px!important}.ipcs{background-image:url(/static/images/background.webp);background-repeat:no-repeat;background-size:cover}.copyleft{display:inline-block;transform:rotate(180deg)}span.search-callsign{background:url(/static/images/search-callsign.svg) no-repeat 0 0;background-size:contain;cursor:pointer;display:inline-block;height:16px;width:20px}#input-group-callsign{margin-bottom:.5rem;margin-right:1rem}#collapseFilters{background-color:#dde2e6;margin-top:10px}#spotsTable{margin-top:10px}#band{margin-top:5px}#dashboard{gap:10px;padding:10px}#telnet-thead{position:sticky;top:0}#chart-band_activity{height:400px;width:100%}.spider_chart{height:480px;width:600px}#silo-propagation-img{height:auto;width:95%}#btn-back-to-top{bottom:20px;display:none;opacity:.7;position:fixed;right:20px}.icon{float:right;font-size:500%;opacity:.16;position:absolute;right:-.3rem;top:0}.grey-dark{background:#495057;color:#efefef}.no-good2{background:linear-gradient(180deg,#cf5252,#790909 80%);color:#fff}.no-good{background:#a83b3b;color:#fff}.good{background:#1f7205;color:#fff}.medium{background:#ffc241;color:#495057}.kpi-card{border-radius:0;box-shadow:1px 1px 3px rgba(0,0,0,.75);display:inline-block;font-family:sans-serif;height:170px;margin-left:.5em;margin-top:.5em;min-height:80px;min-width:80px;overflow:hidden;padding:1em;position:relative;width:170px}.card-value{display:block;font-size:200%;font-weight:bolder}.card-text{display:block;font-size:100%;padding-left:.2em}#propagation-wrapper{display:flex;flex-wrap:wrap;width:100%}#card-container{display:flex;flex-wrap:wrap;width:80%}#muf-container{margin:auto}#solar-data-updated-txt{font-size:14px}.ts-wrapper.multi .ts-control .item{background:#15539e;border-radius:3px;color:#fff}
@import url("https://cdn.jsdelivr.net/npm/bootstrap-icons@1.10.3/font/bootstrap-icons.css");@font-face{font-display:swap;font-family:bootstrap-icons}.badge-responsive{width:70px}@media screen and (max-width:768px){.text-responsive{font-size:12px}.badge-responsive{width:40px}#collapseFilters.collapsing{position:absolute!important;z-index:20}#collapseFilters.collapse.show{display:block;position:absolute;z-index:20}.navbar-collapse{max-height:none!important}}#form-filters{max-width:360px}.img-flag{background-color:#fff;background-size:cover!important;border:1px solid #ddd;border-radius:2px;-webkit-box-shadow:0 2px 10px rgba(0,0,0,.5),0 2px 3px rgba(0,0,0,.5);-moz-box-shadow:0 2px 10px rgba(0,0,0,.5),0 2px 3px rgba(0,0,0,.5);-o-box-shadow:0 2px 10px rgba(0,0,0,.5),0 2px 3px rgba(0,0,0,.5);box-shadow:0 2px 10px rgba(0,0,0,.5),0 2px 3px rgba(0,0,0,.5);height:19px!important;max-height:auto;max-width:auto;padding:3px;width:32px!important}.ipcs{background-image:url(/static/images/background.webp);background-repeat:no-repeat;background-size:cover}.copyleft{display:inline-block;transform:rotate(180deg)}span.search-callsign{background:url(/static/images/search-callsign.svg) no-repeat 0 0;background-size:contain;cursor:pointer;display:inline-block;height:16px;width:20px}#input-group-callsign{margin-bottom:.5rem;margin-right:1rem}#collapseFilters{background-color:#dde2e6;margin-top:10px}#spotsTable{margin-top:10px}#band{margin-top:5px}#dashboard{gap:10px;padding:10px}#telnet-thead{position:sticky;top:0}#chart-band_activity{height:400px;width:100%}.spider_chart{height:480px;width:600px}#silo-propagation-img{height:auto;width:95%}#btn-back-to-top{bottom:20px;display:none;opacity:.7;position:fixed;right:20px}.icon{float:right;font-size:500%;opacity:.16;position:absolute;right:-.3rem;top:0}.grey-dark{background:#495057;color:#efefef}.no-good2{background:linear-gradient(180deg,#cf5252,#790909 80%);color:#fff}.no-good{background:#a83b3b;color:#fff}.good{background:#1f7205;color:#fff}.medium{background:#ffc241;color:#495057}.kpi-card{border-radius:0;box-shadow:1px 1px 3px rgba(0,0,0,.75);display:inline-block;font-family:sans-serif;height:170px;margin-left:.5em;margin-top:.5em;min-height:80px;min-width:80px;overflow:hidden;padding:1em;position:relative;width:170px}.card-value{display:block;font-size:180%;font-weight:bolder}.card-text{display:block;font-size:100%;padding-left:.2em}@media only screen and (max-width:768px){.kpi-card{height:120px!important;width:120px!important}.card-value{display:block;font-size:150%;font-weight:bolder}.card-text{display:block;font-size:80%;padding-left:.2em}}#propagation-wrapper{display:flex;flex-wrap:wrap;width:100%}#card-container{display:flex;flex-wrap:wrap;width:80%}#muf-container{margin:auto}#solar-data-updated-txt{font-size:14px}.ts-wrapper.multi .ts-control .item{background:#15539e;border-radius:3px;color:#fff}

1175
static/data/cty_wt_mod.dat
View File

File diff suppressed because it is too large Load Diff

14
static/js/dev/cookie_consent.js
View File

@ -1,14 +0,0 @@
/*
script used to acquire user consent to cookie banner (and set the cookie consent)
*/
let cookie_modal = new bootstrap.Modal(document.getElementById('cookie_consent_modal'), {
keyboard: false
});
cookie_modal.show();
//if button is pressed, setting cookie
document.getElementById('cookie_consent_btn').onclick = function(){
setCookie('cookie_consent',true,30);
cookie_modal.hide();
};

1
static/js/rel/cookie_consent.min.js vendored
View File

@ -1 +0,0 @@
let cookie_modal=new bootstrap.Modal(document.getElementById("cookie_consent_modal"),{keyboard:!1});cookie_modal.show(),document.getElementById("cookie_consent_btn").onclick=function(){setCookie("cookie_consent",!0,30),cookie_modal.hide()};

2
static/pwa/manifest.webmanifest
View File

@ -1,5 +1,5 @@
{
"name": "IU1BOW Spiderweb v2.5.2",
"name": "IU1BOW Spiderweb v2.5.3",
"description": "DXCluser for ham radio by IU1BOW",
"short_name": "Spiderweb",
"theme_color": "#f3b221",

28
static/pwa/service-worker.js
View File

@ -1,5 +1,5 @@
// Dichiarazione della costante per il nome della cache
const CACHE_NAME = 'pwa-spiderweb_v2.5.2'
const CACHE_NAME = 'pwa-spiderweb_v2.5.3'
// Dichiarazione della costante per gli URL da mettere in cache
const URLS_TO_CACHE = [
@ -27,7 +27,6 @@ const URLS_TO_CACHE = [
'/cookies.html'
];
// Install
self.addEventListener('install', event => {
event.waitUntil(
@ -52,7 +51,7 @@ self.addEventListener('activate', event => {
})
);
});
/*
//Managing request
self.addEventListener('fetch', event => {
console.log(event.request.url);
@ -85,4 +84,25 @@ self.addEventListener('fetch', event => {
});
})
);
});
});
*/
// Nel tuo service worker
self.addEventListener('fetch', (event) => {
event.respondWith(networkFirstCacheFallback(event.request));
});
async function networkFirstCacheFallback(request) {
try {
// Prova a recuperare la risposta dalla rete
const networkResponse = await fetch(request);
// Se la risposta dalla rete è valida, restituiscila
return networkResponse;
} catch (error) {
// Se la rete fallisce, prova a recuperare la risposta dalla cache
const cacheResponse = await caches.match(request);
// Restituisci l'entry dalla cache (fallback)
return cacheResponse;
}
}

31
templates/_base.html
View File

@ -96,7 +96,7 @@
<span class="copyleft">&copy;</span> Copyleft:
<span id="copyDate"></span>
<a href="https://github.com/coulisse/spiderweb/" target="blank" rel="noopener">IU1BOW - Spiderweb</a>
<span id="version">v2.5.2</span>
<span id="version">v2.5.3</span>
</div>
</footer>
<script async src="static/js/rel/load-sw.min.js"></script>
@ -123,35 +123,6 @@
{% endblock app_scripts %}
{% block inline_scripts %}
{% endblock inline_scripts %}
<!-- cookie consent management -->
{% block cookie %}
{% if cookies_check() %}
{# then user has already consented so no requirement for consent banner #}
{% else %}
{# show a cookie consent banner #}
<!-- Modal for cookie consent-->
<div class="modal fade" id="cookie_consent_modal" tabindex="-1" aria-labelledby="cookie-consent-container"
aria-hidden="true">
<div class="modal-dialog">
<div class="modal-content">
<div class="modal-header">
<strong class="modal-title" id="exampleModalLabel">We use cookies</strong>
</div>
<div class="modal-body">
<p>We use only technical cookies.</p>
<p>Clicking &quot;I agree&quot;, you agree to the storing of cookies on your device. To learn more
about how we use cookies, please see our cookies policy.</p>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-primary" id="cookie_consent_btn">I agree</button>
</div>
</div>
</div>
</div>
<script defer src="static/js/rel/cookie_consent.min.js"></script>
{% endif %}
{% endblock cookie %}
<!-- Back to top button -->
<button type="button" class="btn btn-secondary btn-floating btn-lg" id="btn-back-to-top">
<i class="bi bi-arrow-up"></i>

51
templates/cookies.html
View File

@ -16,29 +16,34 @@
{% endblock %}
{% block contents %}
<div class="col mr-3 px-2">
<p class="text-justify"><span class="font-weight-bold">Cookies</span> are small text files that can be used by
websites to make a user's experience more efficient. This site uses different types of cookies. You can at any time
change or withdraw
your consent from the Cookies page on my website. Some cookies are placed by third party services that appear on our
pages, for example if you view or listen to any embedded audio or video content. I don't control the setting of
these cookies, so
please check the websites of these third parties for more information about their cookies and how they manage them.
</p>
<p class="text-justify"><span class="font-weight-bold">Necessary</span> cookies help make a website usable by enabling
basic functions like page navigation and access to secure areas of the website. The website cannot function properly
without
these cookies.</p>
<p class="text-justify"><span class="font-weight-bold">Preference</span> cookies enable a website to remember
information that changes the way the website behaves or looks, like your preferred language or the region that you
are in.</p>
<p class="text-justify"><span class="font-weight-bold">Statistic</span> cookies help website owners to understand how
visitors interact with websites by collecting and reporting information anonymously.</p>
<p class="text-justify"><span class="font-weight-bold">Marketing</span> cookies are used to track visitors across
websites. The intention is to display content, as well as ads that are relevant and engaging for the individual user
and thereby more
valuable for publishers and third party advertisers.</p>
<p class="text-justify"><span class="font-weight-bold">Unclassified</span> cookies are cookies that we are in the
process of classifying, together with the providers of individual cookies.</p>
<h1>What Are Cookies?</h1>
<p>Cookies, also known as internet cookies, are small text files that websites generate and send to your web browser when you visit them. These cookies serve various purposes and play a crucial role in enhancing your browsing experience.</p>
<p>They can be categorized based on their lifespan and origin:</p>
<ul>
<li><strong>Session Cookies:</strong> Exist only during your browsing session and are deleted when you close your browser.</li>
<li><strong>Persistent Cookies:</strong> Remain on your device for a specified period (e.g., days, months, or years).</li>
<li><strong>First-Party Cookies:</strong> Set by the website you're currently visiting.</li>
<li><strong>Third-Party Cookies:</strong> Set by external domains (e.g., advertisers, analytics services) embedded in the website.</li>
</ul>
<h1>Strictly Necessary Cookies</h1>
<p>Strictly necessary cookies (also known as essential cookies) are essential for the basic functioning of a website. They ensure a seamless consumer experience and data security. These cookies do not track user behavior for analytics or advertising purposes.</p>
<p>Examples of what strictly necessary cookies do:</p>
<ul>
<li>Authentication: Verify consumer identities and provide secure account access.</li>
<li>Session Management: Maintain consumers' browsing activities in a single session.</li>
<li>Security: Protect the site and consumers from malicious activities.</li>
<li>Consumer Interface Preferences: Remember choices like language or font size.</li>
</ul>
<div class="alert alert-info mt-4">
<strong>Cookie Disclaimer:</strong> This website uses <em><strong>strictly necessary technical cookies</strong></em> to enhance your browsing experience. These cookies are essential for the proper functioning of the site and do not track or store any personal information. By continuing to use this website, you consent to the use of these cookies.
</div>
</div>
{% endblock %}
{% block app_data %}

221
webapp.py
View File

@ -15,6 +15,7 @@ from lib.cty import prefix_table
from lib.plot_data_provider import ContinentsBandsProvider, SpotsPerMounthProvider, SpotsTrend, HourBand, WorldDxSpotsLive
import requests
import xmltodict
from lib.qry_builder import query_build, query_build_callsign, query_build_callsing_list
logging.config.fileConfig("cfg/webapp_log_config.ini", disable_existing_loggers=True)
logger = logging.getLogger(__name__)
@ -77,199 +78,6 @@ pfxt = prefix_table()
# create object query manager
qm = query_manager()
# find id in json : ie frequency / continent
def find_id_json(json_object, name):
return [obj for obj in json_object if obj["id"] == name][0]
def query_build_callsign(callsign):
query_string = ""
if len(callsign) <= 14:
query_string = (
"(SELECT rowid, spotter AS de, freq, spotcall AS dx, comment AS comm, time, spotdxcc from spot WHERE spotter='"
+ callsign
+ "'"
)
query_string += " ORDER BY rowid desc limit 10)"
query_string += " UNION "
query_string += (
"(SELECT rowid, spotter AS de, freq, spotcall AS dx, comment AS comm, time, spotdxcc from spot WHERE spotcall='"
+ callsign
+ "'"
)
query_string += " ORDER BY rowid desc limit 10);"
else:
logging.warning("callsign too long")
return query_string
def query_build(parameters):
try:
last_rowid = str(parameters["lr"]) # Last rowid fetched by front end
get_param = lambda parameters, parm_name: parameters[parm_name] if (parm_name in parameters) else []
dxcalls=get_param(parameters, "dxcalls")
band=get_param(parameters, "band")
dere=get_param(parameters, "de_re")
dxre=get_param(parameters, "dx_re")
mode=get_param(parameters, "mode")
exclft8=get_param(parameters, "exclft8")
exclft4=get_param(parameters, "exclft4")
decq = []
if "cqdeInput" in parameters:
decq[0] = parameters["cqdeInput"]
dxcq = []
if "cqdxInput" in parameters:
dxcq[0] = parameters["cqdxInput"]
query_string = ""
#construct callsign of spot dx callsign
dxcalls_qry_string = " AND spotcall IN (" + ''.join(map(lambda x: "'" + x + "'," if x != dxcalls[-1] else "'" + x + "'", dxcalls)) + ")"
# construct band query decoding frequencies with json file
band_qry_string = " AND (("
for i, item_band in enumerate(band):
freq = find_id_json(band_frequencies["bands"], item_band)
if i > 0:
band_qry_string += ") OR ("
band_qry_string += (
"freq BETWEEN " + str(freq["min"]) + " AND " + str(freq["max"])
)
band_qry_string += "))"
# construct mode query
mode_qry_string = " AND (("
for i, item_mode in enumerate(mode):
single_mode = find_id_json(modes_frequencies["modes"], item_mode)
if i > 0:
mode_qry_string += ") OR ("
for j in range(len(single_mode["freq"])):
if j > 0:
mode_qry_string += ") OR ("
mode_qry_string += (
"freq BETWEEN "
+ str(single_mode["freq"][j]["min"])
+ " AND "
+ str(single_mode["freq"][j]["max"])
)
mode_qry_string += "))"
#Exluding FT8 or FT4 connection
ft8_qry_string = " AND ("
if exclft8:
ft8_qry_string += "(comment NOT LIKE '%FT8%')"
single_mode = find_id_json(modes_frequencies["modes"], "digi-ft8")
for j in range(len(single_mode["freq"])):
ft8_qry_string += (
" AND (freq NOT BETWEEN "
+ str(single_mode["freq"][j]["min"])
+ " AND "
+ str(single_mode["freq"][j]["max"])
+ ")"
)
ft8_qry_string += ")"
ft4_qry_string = " AND ("
if exclft4:
ft4_qry_string += "(comment NOT LIKE '%FT4%')"
single_mode = find_id_json(modes_frequencies["modes"], "digi-ft4")
for j in range(len(single_mode["freq"])):
ft4_qry_string += (
" AND (freq NOT BETWEEN "
+ str(single_mode["freq"][j]["min"])
+ " AND "
+ str(single_mode["freq"][j]["max"])
+ ")"
)
ft4_qry_string += ")"
# construct DE continent region query
dere_qry_string = " AND spottercq IN ("
for i, item_dere in enumerate(dere):
continent = find_id_json(continents_cq["continents"], item_dere)
if i > 0:
dere_qry_string += ","
dere_qry_string += str(continent["cq"])
dere_qry_string += ")"
# construct DX continent region query
dxre_qry_string = " AND spotcq IN ("
for i, item_dxre in enumerate(dxre):
continent = find_id_json(continents_cq["continents"], item_dxre)
if i > 0:
dxre_qry_string += ","
dxre_qry_string += str(continent["cq"])
dxre_qry_string += ")"
if enable_cq_filter == "Y":
# construct de cq query
decq_qry_string = ""
if len(decq) == 1:
if decq[0].isnumeric():
decq_qry_string = " AND spottercq =" + decq[0]
# construct dx cq query
dxcq_qry_string = ""
if len(dxcq) == 1:
if dxcq[0].isnumeric():
dxcq_qry_string = " AND spotcq =" + dxcq[0]
if last_rowid is None:
last_rowid = "0"
if not last_rowid.isnumeric():
last_rowid = 0
query_string = (
"SELECT rowid, spotter AS de, freq, spotcall AS dx, comment AS comm, time, spotdxcc from spot WHERE rowid > "
+ last_rowid
)
if dxcalls:
query_string += dxcalls_qry_string
if len(band) > 0:
query_string += band_qry_string
if len(mode) > 0:
query_string += mode_qry_string
if exclft8:
query_string += ft8_qry_string
if exclft4:
query_string += ft4_qry_string
if len(dere) > 0:
query_string += dere_qry_string
if len(dxre) > 0:
query_string += dxre_qry_string
if enable_cq_filter == "Y":
if len(decq_qry_string) > 0:
query_string += decq_qry_string
if len(dxcq_qry_string) > 0:
query_string += dxcq_qry_string
query_string += " ORDER BY rowid desc limit 50;"
logger.debug (query_string)
except Exception as e:
logger.error(e)
query_string = ""
return query_string
# the main query to show spots
# it gets url parameter in order to apply the build the right query
# and apply the filter required. It returns a json with the spots
@ -278,10 +86,10 @@ def spotquery(parameters):
if 'callsign' in parameters:
logging.debug('search callsign')
query_string = query_build_callsign( parameters['callsign'] )
query_string = query_build_callsign(logger,parameters['callsign'] )
else:
logging.debug('search eith other filters')
query_string = query_build(parameters)
query_string = query_build(logger,parameters,band_frequencies,modes_frequencies,continents_cq,enable_cq_filter)
qm.qry(query_string)
data = qm.get_data()
@ -377,7 +185,7 @@ def spots():
#Show all dx spot callsigns
def get_dx_calls():
try:
query_string = "SELECT spotcall AS dx FROM (select spotcall from spot order by rowid desc limit 50000) s1 GROUP BY spotcall ORDER BY count(spotcall) DESC LIMIT 100;"
query_string = query_build_callsing_list
qm.qry(query_string)
data = qm.get_data()
row_headers = qm.get_headers()
@ -425,7 +233,6 @@ def plots():
)
return response
@app.route("/propagation.html")
def propagation():
@ -453,6 +260,8 @@ def propagation():
solar_data=solar_data
)
)
#response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
return response
@app.route("/cookies.html", methods=["GET"])
@ -581,18 +390,6 @@ def csp_reports():
response=flask.Response(status=204)
return response
@app.context_processor
def inject_template_scope():
injections = dict()
def cookies_check():
value = request.cookies.get("cookie_consent")
return value == "true"
injections.update(cookies_check=cookies_check)
return injections
@app.after_request
def add_security_headers(resp):
@ -602,7 +399,8 @@ def add_security_headers(resp):
resp.headers["X-Content-Type-Options"] = "nosniff"
resp.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
#resp.headers["Access-Control-Allow-Origin"]= "sidc.be prop.kc2g.com www.hamqsl.com"
resp.headers["Cache-Control"] = "public, no-cache"
#resp.headers["Cache-Control"] = "public, no-cache"
resp.headers["Cache-Control"] = "public, no-cache, must-revalidate, max-age=900"
resp.headers["Pragma"] = "no-cache"
resp.headers["ETag"] = app.config["VERSION"]
#resp.headers["Report-To"] = '{"group":"csp-endpoint", "max_age":10886400, "endpoints":[{"url":"/csp-reports"}]}'
@ -627,6 +425,5 @@ def add_security_headers(resp):
#report-to csp-endpoint;\
#script-src 'self' cdnjs.cloudflare.com cdn.jsdelivr.net 'nonce-sedfGFG32xs';\
#script-src 'self' cdnjs.cloudflare.com cdn.jsdelivr.net 'nonce-"+inline_script_nonce+"';\
if __name__ == "__main__":
app.run(host="0.0.0.0")
app.run(host="0.0.0.0")

611
webapp.py.old
View File

@ -1,611 +0,0 @@
__author__ = "IU1BOW - Corrado"
import flask
import secrets
from flask import request, render_template
from flask_wtf.csrf import CSRFProtect
from flask_minify import minify
import json
import threading
import logging
import logging.config
from lib.dxtelnet import who
from lib.adxo import get_adxo_events
from lib.qry import query_manager
from lib.cty import prefix_table
from lib.plot_data_provider import ContinentsBandsProvider, SpotsPerMounthProvider, SpotsTrend, HourBand, WorldDxSpotsLive
logging.config.fileConfig("cfg/webapp_log_config.ini", disable_existing_loggers=True)
logger = logging.getLogger(__name__)
logger.info("Starting SPIDERWEB")
app = flask.Flask(__name__)
app.config["SECRET_KEY"] = secrets.token_hex(16)
app.config.update(
SESSION_COOKIE_SECURE=True,
SESSION_COOKIE_HTTPONLY=False,
SESSION_COOKIE_SAMESITE="Strict",
)
version_file = open("cfg/version.txt", "r")
app.config["VERSION"] = version_file.read().strip()
version_file.close
logger.info("Version:"+app.config["VERSION"] )
inline_script_nonce = ""
csrf = CSRFProtect(app)
logger.debug(app.config)
if app.config["DEBUG"]:
minify(app=app, html=False, js=False, cssless=False)
else:
minify(app=app, html=True, js=True, cssless=False)
# load config file
with open("cfg/config.json") as json_data_file:
cfg = json.load(json_data_file)
logging.debug("CFG:")
logging.debug(cfg)
# load bands file
with open("cfg/bands.json") as json_bands:
band_frequencies = json.load(json_bands)
# load mode file
with open("cfg/modes.json") as json_modes:
modes_frequencies = json.load(json_modes)
# load continents-cq file
with open("cfg/continents.json") as json_continents:
continents_cq = json.load(json_continents)
# read and set default for enabling cq filter
if cfg.get("enable_cq_filter"):
enable_cq_filter = cfg["enable_cq_filter"].upper()
else:
enable_cq_filter = "N"
# define country table for search info on callsigns
pfxt = prefix_table()
# create object query manager
qm = query_manager()
# find id in json : ie frequency / continent
def find_id_json(json_object, name):
return [obj for obj in json_object if obj["id"] == name][0]
def query_build_callsign(callsign):
query_string = ""
if len(callsign) <= 14:
query_string = (
"(SELECT rowid, spotter AS de, freq, spotcall AS dx, comment AS comm, time, spotdxcc from spot WHERE spotter='"
+ callsign
+ "'"
)
query_string += " ORDER BY rowid desc limit 10)"
query_string += " UNION "
query_string += (
"(SELECT rowid, spotter AS de, freq, spotcall AS dx, comment AS comm, time, spotdxcc from spot WHERE spotcall='"
+ callsign
+ "'"
)
query_string += " ORDER BY rowid desc limit 10);"
else:
logging.warning("callsign too long")
return query_string
def query_build(parameters):
try:
last_rowid = str(parameters["lr"]) # Last rowid fetched by front end
get_param = lambda parameters, parm_name: parameters[parm_name] if (parm_name in parameters) else []
dxcalls=get_param(parameters, "dxcalls")
band=get_param(parameters, "band")
dere=get_param(parameters, "de_re")
dxre=get_param(parameters, "dx_re")
mode=get_param(parameters, "mode")
exclft8=get_param(parameters, "exclft8")
exclft4=get_param(parameters, "exclft4")
decq = []
if "cqdeInput" in parameters:
decq[0] = parameters["cqdeInput"]
dxcq = []
if "cqdxInput" in parameters:
dxcq[0] = parameters["cqdxInput"]
query_string = ""
#construct callsign of spot dx callsign
dxcalls_qry_string = " AND spotcall IN (" + ''.join(map(lambda x: "'" + x + "'," if x != dxcalls[-1] else "'" + x + "'", dxcalls)) + ")"
# construct band query decoding frequencies with json file
band_qry_string = " AND (("
for i, item_band in enumerate(band):
freq = find_id_json(band_frequencies["bands"], item_band)
if i > 0:
band_qry_string += ") OR ("
band_qry_string += (
"freq BETWEEN " + str(freq["min"]) + " AND " + str(freq["max"])
)
band_qry_string += "))"
# construct mode query
mode_qry_string = " AND (("
for i, item_mode in enumerate(mode):
single_mode = find_id_json(modes_frequencies["modes"], item_mode)
if i > 0:
mode_qry_string += ") OR ("
for j in range(len(single_mode["freq"])):
if j > 0:
mode_qry_string += ") OR ("
mode_qry_string += (
"freq BETWEEN "
+ str(single_mode["freq"][j]["min"])
+ " AND "
+ str(single_mode["freq"][j]["max"])
)
mode_qry_string += "))"
#Exluding FT8 or FT4 connection
ft8_qry_string = " AND ("
if exclft8:
ft8_qry_string += "(comment NOT LIKE '%FT8%')"
single_mode = find_id_json(modes_frequencies["modes"], "digi-ft8")
for j in range(len(single_mode["freq"])):
ft8_qry_string += (
" AND (freq NOT BETWEEN "
+ str(single_mode["freq"][j]["min"])
+ " AND "
+ str(single_mode["freq"][j]["max"])
+ ")"
)
ft8_qry_string += ")"
ft4_qry_string = " AND ("
if exclft4:
ft4_qry_string += "(comment NOT LIKE '%FT4%')"
single_mode = find_id_json(modes_frequencies["modes"], "digi-ft4")
for j in range(len(single_mode["freq"])):
ft4_qry_string += (
" AND (freq NOT BETWEEN "
+ str(single_mode["freq"][j]["min"])
+ " AND "
+ str(single_mode["freq"][j]["max"])
+ ")"
)
ft4_qry_string += ")"
# construct DE continent region query
dere_qry_string = " AND spottercq IN ("
for i, item_dere in enumerate(dere):
continent = find_id_json(continents_cq["continents"], item_dere)
if i > 0:
dere_qry_string += ","
dere_qry_string += str(continent["cq"])
dere_qry_string += ")"
# construct DX continent region query
dxre_qry_string = " AND spotcq IN ("
for i, item_dxre in enumerate(dxre):
continent = find_id_json(continents_cq["continents"], item_dxre)
if i > 0:
dxre_qry_string += ","
dxre_qry_string += str(continent["cq"])
dxre_qry_string += ")"
if enable_cq_filter == "Y":
# construct de cq query
decq_qry_string = ""
if len(decq) == 1:
if decq[0].isnumeric():
decq_qry_string = " AND spottercq =" + decq[0]
# construct dx cq query
dxcq_qry_string = ""
if len(dxcq) == 1:
if dxcq[0].isnumeric():
dxcq_qry_string = " AND spotcq =" + dxcq[0]
if last_rowid is None:
last_rowid = "0"
if not last_rowid.isnumeric():
last_rowid = 0
query_string = (
"SELECT rowid, spotter AS de, freq, spotcall AS dx, comment AS comm, time, spotdxcc from spot WHERE rowid > "
+ last_rowid
)
if dxcalls:
query_string += dxcalls_qry_string
if len(band) > 0:
query_string += band_qry_string
if len(mode) > 0:
query_string += mode_qry_string
if exclft8:
query_string += ft8_qry_string
if exclft4:
query_string += ft4_qry_string
if len(dere) > 0:
query_string += dere_qry_string
if len(dxre) > 0:
query_string += dxre_qry_string
if enable_cq_filter == "Y":
if len(decq_qry_string) > 0:
query_string += decq_qry_string
if len(dxcq_qry_string) > 0:
query_string += dxcq_qry_string
query_string += " ORDER BY rowid desc limit 50;"
logger.debug (query_string)
except Exception as e:
logger.error(e)
query_string = ""
return query_string
# the main query to show spots
# it gets url parameter in order to apply the build the right query
# and apply the filter required. It returns a json with the spots
def spotquery(parameters):
try:
if 'callsign' in parameters:
logging.debug('search callsign')
query_string = query_build_callsign( parameters['callsign'] )
else:
logging.debug('search eith other filters')
query_string = query_build(parameters)
qm.qry(query_string)
data = qm.get_data()
row_headers = qm.get_headers()
logger.debug("query done")
logger.debug(data)
if data is None or len(data) == 0:
logger.warning("no data found")
payload = []
for result in data:
# create dictionary from recorset
main_result = dict(zip(row_headers, result))
# find the country in prefix table
search_prefix = pfxt.find(main_result["dx"])
# merge recordset and contry prefix
main_result["country"] = search_prefix["country"]
main_result["iso"] = search_prefix["iso"]
payload.append({**main_result})
return payload
except Exception as e:
logger.error(e)
# find adxo events
adxo_events = None
def get_adxo():
global adxo_events
adxo_events = get_adxo_events()
threading.Timer(12 * 3600, get_adxo).start()
get_adxo()
# create data provider for charts
heatmap_cbp = ContinentsBandsProvider(logger, qm, continents_cq, band_frequencies)
bar_graph_spm = SpotsPerMounthProvider(logger, qm)
line_graph_st = SpotsTrend(logger, qm)
bubble_graph_hb = HourBand(logger, qm, band_frequencies)
geo_graph_wdsl = WorldDxSpotsLive(logger, qm, pfxt)
# ROUTINGS
@app.route("/spotlist", methods=["POST"])
@csrf.exempt
def spotlist():
logger.debug(request.json)
response = flask.Response(json.dumps(spotquery(request.json)))
return response
def who_is_connected():
host=cfg["telnet"]["host"]
port=cfg["telnet"]["port"]
user=cfg["telnet"]["user"]
password=cfg["telnet"]["password"]
response = who(host, port, user, password)
logger.debug("list of connected clusters:")
logger.debug(response)
return response
#Calculate nonce token used in inline script and in csp "script-src" header
def get_nonce():
global inline_script_nonce
inline_script_nonce = secrets.token_hex()
return inline_script_nonce
@app.route("/", methods=["GET"])
@app.route("/index.html", methods=["GET"])
def spots():
response = flask.Response(
render_template(
"index.html",
inline_script_nonce=get_nonce(),
mycallsign=cfg["mycallsign"],
telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
mail=cfg["mail"],
menu_list=cfg["menu"]["menu_list"],
enable_cq_filter=enable_cq_filter,
timer_interval=cfg["timer"]["interval"],
adxo_events=adxo_events,
continents=continents_cq,
bands=band_frequencies,
dx_calls=get_dx_calls(),
)
)
return response
#Show all dx spot callsigns
def get_dx_calls():
try:
query_string = "SELECT spotcall AS dx FROM spot GROUP BY spotcall ORDER BY count(spotcall) DESC LIMIT 500"
qm.qry(query_string)
data = qm.get_data()
row_headers = qm.get_headers()
payload = []
for result in data:
main_result = dict(zip(row_headers, result))
payload.append(main_result["dx"])
logger.debug("last DX Callsigns:")
logger.debug(payload)
return payload
except Exception as e:
return []
@app.route("/service-worker.js", methods=["GET"])
def sw():
return app.send_static_file("pwa/service-worker.js")
@app.route("/offline.html")
def root():
return app.send_static_file("html/offline.html")
@app.route("/world.json")
def world_data():
return app.send_static_file("data/world.json")
@app.route("/plots.html")
def plots():
whoj = who_is_connected()
response = flask.Response(
render_template(
"plots.html",
inline_script_nonce=get_nonce(),
mycallsign=cfg["mycallsign"],
telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
mail=cfg["mail"],
menu_list=cfg["menu"]["menu_list"],
who=whoj,
continents=continents_cq,
bands=band_frequencies,
)
)
return response
@app.route("/propagation.html")
def propagation():
response = flask.Response(
render_template(
"propagation.html",
inline_script_nonce=get_nonce(),
mycallsign=cfg["mycallsign"],
telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
mail=cfg["mail"],
menu_list=cfg["menu"]["menu_list"],
)
)
return response
@app.route("/cookies.html", methods=["GET"])
def cookies():
response = flask.Response(
render_template(
"cookies.html",
inline_script_nonce=get_nonce(),
mycallsign=cfg["mycallsign"],
telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
mail=cfg["mail"],
menu_list=cfg["menu"]["menu_list"],
)
)
return response
@app.route("/privacy.html", methods=["GET"])
def privacy():
response = flask.Response(
render_template(
"privacy.html",
inline_script_nonce=get_nonce(),
mycallsign=cfg["mycallsign"],
telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
mail=cfg["mail"],
menu_list=cfg["menu"]["menu_list"],
)
)
return response
@app.route("/sitemap.xml")
def sitemap():
return app.send_static_file("sitemap.xml")
@app.route("/callsign.html", methods=["GET"])
def callsign():
# payload=spotquery()
callsign = request.args.get("c")
response = flask.Response(
render_template(
"callsign.html",
inline_script_nonce=get_nonce(),
mycallsign=cfg["mycallsign"],
telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
mail=cfg["mail"],
menu_list=cfg["menu"]["menu_list"],
timer_interval=cfg["timer"]["interval"],
callsign=callsign,
adxo_events=adxo_events,
continents=continents_cq,
bands=band_frequencies,
)
)
return response
# API that search a callsign and return all informations about that
@app.route("/callsign", methods=["GET"])
def find_callsign():
callsign = request.args.get("c")
response = pfxt.find(callsign)
if response is None:
response = flask.Response(status=204)
return response
@app.route("/plot_get_heatmap_data", methods=["POST"])
@csrf.exempt
def get_heatmap_data():
#continent = request.args.get("continent")
continent = request.json['continent']
logger.debug(request.get_json());
response = flask.Response(json.dumps(heatmap_cbp.get_data(continent)))
logger.debug(response)
if response is None:
response = flask.Response(status=204)
return response
@app.route("/plot_get_dx_spots_per_month", methods=["POST"])
@csrf.exempt
def get_dx_spots_per_month():
response = flask.Response(json.dumps(bar_graph_spm.get_data()))
logger.debug(response)
if response is None:
response = flask.Response(status=204)
return response
@app.route("/plot_get_dx_spots_trend", methods=["POST"])
@csrf.exempt
def get_dx_spots_trend():
response = flask.Response(json.dumps(line_graph_st.get_data()))
logger.debug(response)
if response is None:
response = flask.Response(status=204)
return response
@app.route("/plot_get_hour_band", methods=["POST"])
@csrf.exempt
def get_dx_hour_band():
response = flask.Response(json.dumps(bubble_graph_hb.get_data()))
logger.debug(response)
if response is None:
response = flask.Response(status=204)
return response
@app.route("/plot_get_world_dx_spots_live", methods=["POST"])
@csrf.exempt
def get_world_dx_spots_live():
response = flask.Response(json.dumps(geo_graph_wdsl.get_data()))
logger.debug(response)
if response is None:
response = flask.Response(status=204)
return response
@app.route("/csp-reports", methods=['POST'])
@csrf.exempt
def csp_reports():
report_data = request.get_data(as_text=True)
logger.warning("CSP Report:")
logger.warning(report_data)
response=flask.Response(status=204)
return response
@app.context_processor
def inject_template_scope():
injections = dict()
def cookies_check():
value = request.cookies.get("cookie_consent")
return value == "true"
injections.update(cookies_check=cookies_check)
return injections
@app.after_request
def add_security_headers(resp):
resp.headers["Strict-Transport-Security"] = "max-age=1000"
resp.headers["X-Xss-Protection"] = "1; mode=block"
resp.headers["X-Frame-Options"] = "SAMEORIGIN"
resp.headers["X-Content-Type-Options"] = "nosniff"
resp.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
resp.headers["Cache-Control"] = "public, no-cache"
resp.headers["Pragma"] = "no-cache"
resp.headers["ETag"] = app.config["VERSION"]
#resp.headers["Report-To"] = '{"group":"csp-endpoint", "max_age":10886400, "endpoints":[{"url":"/csp-reports"}]}'
resp.headers["Content-Security-Policy"] = "\
default-src 'self';\
script-src 'self' cdnjs.cloudflare.com cdn.jsdelivr.net 'nonce-"+inline_script_nonce+"';\
style-src 'self' cdnjs.cloudflare.com cdn.jsdelivr.net;\
object-src 'none';base-uri 'self';\
connect-src 'self' cdn.jsdelivr.net cdnjs.cloudflare.com sidc.be;\
font-src 'self' cdn.jsdelivr.net;\
frame-src 'self';\
frame-ancestors 'none';\
form-action 'none';\
img-src 'self' data: cdnjs.cloudflare.com sidc.be;\
manifest-src 'self';\
media-src 'self';\
worker-src 'self';\
report-uri /csp-reports;\
"
return resp
#report-to csp-endpoint;\
#script-src 'self' cdnjs.cloudflare.com cdn.jsdelivr.net 'nonce-sedfGFG32xs';\
#script-src 'self' cdnjs.cloudflare.com cdn.jsdelivr.net 'nonce-"+inline_script_nonce+"';\
if __name__ == "__main__":
app.run(host="0.0.0.0")

617
webapp_sanit.py.old
View File

@ -1,617 +0,0 @@
__author__ = "IU1BOW - Corrado"
import flask
import secrets
from flask import request, render_template
from flask_wtf.csrf import CSRFProtect
from flask_minify import minify
import json
import threading
import logging
import logging.config
from lib.dxtelnet import who
from lib.adxo import get_adxo_events
from lib.qry import query_manager
from lib.cty import prefix_table
from lib.plot_data_provider import ContinentsBandsProvider, SpotsPerMounthProvider, SpotsTrend, HourBand, WorldDxSpotsLive
logging.config.fileConfig("cfg/webapp_log_config.ini", disable_existing_loggers=True)
logger = logging.getLogger(__name__)
logger.info("Starting SPIDERWEB")
app = flask.Flask(__name__)
app.config["SECRET_KEY"] = secrets.token_hex(16)
app.config.update(
SESSION_COOKIE_SECURE=True,
SESSION_COOKIE_HTTPONLY=False,
SESSION_COOKIE_SAMESITE="Strict",
)
version_file = open("cfg/version.txt", "r")
app.config["VERSION"] = version_file.read().strip()
version_file.close()
logger.info("Version:" + app.config["VERSION"])
inline_script_nonce = ""
csrf = CSRFProtect(app)
logger.debug(app.config)
if app.config["DEBUG"]:
minify(app=app, html=False, js=False, cssless=False)
else:
minify(app=app, html=True, js=True, cssless=False)
# load config file
with open("cfg/config.json") as json_data_file:
cfg = json.load(json_data_file)
logging.debug("CFG:")
logging.debug(cfg)
# load bands file
with open("cfg/bands.json") as json_bands:
band_frequencies = json.load(json_bands)
# load mode file
with open("cfg/modes.json") as json_modes:
modes_frequencies = json.load(json_modes)
# load continents-cq file
with open("cfg/continents.json") as json_continents:
continents_cq = json.load(json_continents)
# read and set default for enabling cq filter
if cfg.get("enable_cq_filter"):
enable_cq_filter = cfg["enable_cq_filter"].upper()
else:
enable_cq_filter = "N"
# define country table for search info on callsigns
pfxt = prefix_table()
# create object query manager
qm = query_manager()
# find id in json : ie frequency / continent
def find_id_json(json_object, name):
return [obj for obj in json_object if obj["id"] == name][0]
def query_build_callsign(callsign):
query_string = ""
if len(callsign) <= 14:
query_string = (
"(SELECT rowid, spotter AS de, freq, spotcall AS dx, comment AS comm, time, spotdxcc from spot WHERE spotter='"
+ callsign
+ "'"
)
query_string += " ORDER BY rowid desc limit 10)"
query_string += " UNION "
query_string += (
"(SELECT rowid, spotter AS de, freq, spotcall AS dx, comment AS comm, time, spotdxcc from spot WHERE spotcall='"
+ callsign
+ "'"
)
query_string += " ORDER BY rowid desc limit 10);"
else:
logging.warning("callsign too long")
return query_string
def query_build(parameters):
try:
last_rowid = str(parameters["lr"]) # Last rowid fetched by front end
get_param = lambda parameters, parm_name: parameters[parm_name] if (parm_name in parameters) else []
dxcalls=get_param(parameters, "dxcalls")
band=get_param(parameters, "band")
dere=get_param(parameters, "de_re")
dxre=get_param(parameters, "dx_re")
mode=get_param(parameters, "mode")
exclft8=get_param(parameters, "exclft8")
exclft4=get_param(parameters, "exclft4")
decq = []
if "cqdeInput" in parameters:
decq.append(parameters["cqdeInput"])
dxcq = []
if "cqdxInput" in parameters:
dxcq.append(parameters["cqdxInput"])
query_string = ""
#construct callsign of spot dx callsign
dxcalls_qry_string = " AND spotcall IN (" + ",".join(["%s" for _ in dxcalls]) + ")"
dxcalls_params = tuple(dxcalls)
# construct band query decoding frequencies with json file
band_qry_string = " AND (("
for i, item_band in enumerate(band):
freq = find_id_json(band_frequencies["bands"], item_band)
if i > 0:
band_qry_string += ") OR ("
band_qry_string += (
"freq BETWEEN " + str(freq["min"]) + " AND " + str(freq["max"])
)
band_qry_string += "))"
# construct mode query
mode_qry_string = " AND (("
for i, item_mode in enumerate(mode):
single_mode = find_id_json(modes_frequencies["modes"], item_mode)
if i > 0:
mode_qry_string += ") OR ("
for j in range(len(single_mode["freq"])):
if j > 0:
mode_qry_string += ") OR ("
mode_qry_string += (
"freq BETWEEN "
+ str(single_mode["freq"][j]["min"])
+ " AND "
+ str(single_mode["freq"][j]["max"])
)
mode_qry_string += "))"
# Exluding FT8 or FT4 connection
ft8_qry_string = " AND ("
if exclft8:
ft8_qry_string += "(comment NOT LIKE '%FT8%')"
single_mode = find_id_json(modes_frequencies["modes"], "digi-ft8")
for j in range(len(single_mode["freq"])):
ft8_qry_string += (
" AND (freq NOT BETWEEN "
+ str(single_mode["freq"][j]["min"])
+ " AND "
+ str(single_mode["freq"][j]["max"])
+ ")"
)
ft8_qry_string += ")"
ft4_qry_string = " AND ("
if exclft4:
ft4_qry_string += "(comment NOT LIKE '%FT4%')"
single_mode = find_id_json(modes_frequencies["modes"], "digi-ft4")
for j in range(len(single_mode["freq"])):
ft4_qry_string += (
" AND (freq NOT BETWEEN "
+ str(single_mode["freq"][j]["min"])
+ " AND "
+ str(single_mode["freq"][j]["max"])
+ ")"
)
ft4_qry_string += ")"
# construct DE continent region query
dere_qry_string = " AND spottercq IN (" + ",".join(["%s" for _ in dere]) + ")"
dere_params = tuple(dere)
# construct DX continent region query
dxre_qry_string = " AND spotcq IN (" + ",".join(["%s" for _ in dxre]) + ")"
dxre_params = tuple(dxre)
if enable_cq_filter == "Y":
# construct de cq query
decq_qry_string = ""
decq_params = []
if len(decq) == 1:
if decq[0].isnumeric():
decq_qry_string = " AND spottercq = %s"
decq_params = (decq[0],)
# construct dx cq query
dxcq_qry_string = ""
dxcq_params = []
if len(dxcq) == 1:
if dxcq[0].isnumeric():
dxcq_qry_string = " AND spotcq = %s"
dxcq_params = (dxcq[0],)
if last_rowid is None:
last_rowid = "0"
if not last_rowid.isnumeric():
last_rowid = 0
query_string = (
"SELECT rowid, spotter AS de, freq, spotcall AS dx, comment AS comm, time, spotdxcc from spot WHERE rowid > %s"
)
if dxcalls:
query_string += dxcalls_qry_string
if len(band) > 0:
query_string += band_qry_string
if len(mode) > 0:
query_string += mode_qry_string
if exclft8:
query_string += ft8_qry_string
if exclft4:
query_string += ft4_qry_string
if len(dere) > 0:
query_string += dere_qry_string
if len(dxre) > 0:
query_string += dxre_qry_string
if enable_cq_filter == "Y":
if len(decq_qry_string) > 0:
query_string += decq_qry_string
decq_params = (decq_params,)
if len(dxcq_qry_string) > 0:
query_string += dxcq_qry_string
dxcq_params = (dxcq_params,)
query_string += " ORDER BY rowid desc limit 50;"
logger.debug(query_string)
except Exception as e:
logger.error(e)
query_string = ""
return query_string, (
last_rowid,
*dxcalls_params,
*band_params,
*mode_params,
*ft8_params,
*ft4_params,
*dere_params,
*dxre_params,
*decq_params,
*dxcq_params,
)
# the main query to show spots
# it gets url parameter in order to apply the build the right query
# and apply the filter required. It returns a json with the spots
def spotquery(parameters):
try:
if 'callsign' in parameters:
logging.debug('search callsign')
query_string, params = query_build_callsign(parameters['callsign'])
else:
logging.debug('search eith other filters')
query_string, params = query_build(parameters)
qm.qry(query_string, params)
data = qm.get_data()
row_headers = qm.get_headers()
logger.debug("query done")
logger.debug(data)
if data is None or len(data) == 0:
logger.warning("no data found")
payload = []
for result in data:
# create dictionary from recorset
main_result = dict(zip(row_headers, result))
# find the country in prefix table
search_prefix = pfxt.find(main_result["dx"])
# merge recordset and contry prefix
main_result["country"] = search_prefix["country"]
main_result["iso"] = search_prefix["iso"]
payload.append({**main_result})
return payload
except Exception as e:
logger.error(e)
# find adxo events
adxo_events = None
def get_adxo():
global adxo_events
adxo_events = get_adxo_events()
threading.Timer(12 * 3600, get_adxo).start()
get_adxo()
# create data provider for charts
heatmap_cbp = ContinentsBandsProvider(logger, qm, continents_cq, band_frequencies)
bar_graph_spm = SpotsPerMounthProvider(logger, qm)
line_graph_st = SpotsTrend(logger, qm)
bubble_graph_hb = HourBand(logger, qm, band_frequencies)
geo_graph_wdsl = WorldDxSpotsLive(logger, qm, pfxt)
# ROUTINGS
@app.route("/spotlist", methods=["POST"])
@csrf.exempt
def spotlist():
logger.debug(request.json)
response = flask.Response(json.dumps(spotquery(request.json)))
return response
def who_is_connected():
host=cfg["telnet"]["host"]
port=cfg["telnet"]["port"]
user=cfg["telnet"]["user"]
password=cfg["telnet"]["password"]
response = who(host, port, user, password)
logger.debug("list of connected clusters:")
logger.debug(response)
return response
#Calculate nonce token used in inline script and in csp "script-src" header
def get_nonce():
global inline_script_nonce
inline_script_nonce = secrets.token_hex()
return inline_script_nonce
@app.route("/", methods=["GET"])
@app.route("/index.html", methods=["GET"])
def spots():
response = flask.Response(
render_template(
"index.html",
inline_script_nonce=get_nonce(),
mycallsign=cfg["mycallsign"],
telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
mail=cfg["mail"],
menu_list=cfg["menu"]["menu_list"],
enable_cq_filter=enable_cq_filter,
timer_interval=cfg["timer"]["interval"],
adxo_events=adxo_events,
continents=continents_cq,
bands=band_frequencies,
dx_calls=get_dx_calls(),
)
)
return response
#Show all dx spot callsigns
def get_dx_calls():
try:
query_string = "SELECT spotcall AS dx FROM spot GROUP BY spotcall ORDER BY count(spotcall) DESC LIMIT 500"
qm.qry(query_string)
data = qm.get_data()
row_headers = qm.get_headers()
payload = []
for result in data:
main_result = dict(zip(row_headers, result))
payload.append(main_result["dx"])
logger.debug("last DX Callsigns:")
logger.debug(payload)
return payload
except Exception as e:
return []
@app.route("/service-worker.js", methods=["GET"])
def sw():
return app.send_static_file("pwa/service-worker.js")
@app.route("/offline.html")
def root():
return app.send_static_file("html/offline.html")
@app.route("/world.json")
def world_data():
return app.send_static_file("data/world.json")
@app.route("/plots.html")
def plots():
whoj = who_is_connected()
response = flask.Response(
render_template(
"plots.html",
inline_script_nonce=get_nonce(),
mycallsign=cfg["mycallsign"],
telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
mail=cfg["mail"],
menu_list=cfg["menu"]["menu_list"],
who=whoj,
continents=continents_cq,
bands=band_frequencies,
)
)
return response
@app.route("/propagation.html")
def propagation():
response = flask.Response(
render_template(
"propagation.html",
inline_script_nonce=get_nonce(),
mycallsign=cfg["mycallsign"],
telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
mail=cfg["mail"],
menu_list=cfg["menu"]["menu_list"],
)
)
return response
@app.route("/cookies.html", methods=["GET"])
def cookies():
response = flask.Response(
render_template(
"cookies.html",
inline_script_nonce=get_nonce(),
mycallsign=cfg["mycallsign"],
telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
mail=cfg["mail"],
menu_list=cfg["menu"]["menu_list"],
)
)
return response
@app.route("/privacy.html", methods=["GET"])
def privacy():
response = flask.Response(
render_template(
"privacy.html",
inline_script_nonce=get_nonce(),
mycallsign=cfg["mycallsign"],
telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
mail=cfg["mail"],
menu_list=cfg["menu"]["menu_list"],
)
)
return response
@app.route("/sitemap.xml")
def sitemap():
return app.send_static_file("sitemap.xml")
@app.route("/callsign.html", methods=["GET"])
def callsign():
# payload=spotquery()
callsign = request.args.get("c")
response = flask.Response(
render_template(
"callsign.html",
inline_script_nonce=get_nonce(),
mycallsign=cfg["mycallsign"],
telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
mail=cfg["mail"],
menu_list=cfg["menu"]["menu_list"],
timer_interval=cfg["timer"]["interval"],
callsign=callsign,
adxo_events=adxo_events,
continents=continents_cq,
bands=band_frequencies,
)
)
return response
# API that search a callsign and return all informations about that
@app.route("/callsign", methods=["GET"])
def find_callsign():
callsign = request.args.get("c")
response = pfxt.find(callsign)
if response is None:
response = flask.Response(status=204)
return response
@app.route("/plot_get_heatmap_data", methods=["POST"])
@csrf.exempt
def get_heatmap_data():
#continent = request.args.get("continent")
continent = request.json['continent']
logger.debug(request.get_json());
response = flask.Response(json.dumps(heatmap_cbp.get_data(continent)))
logger.debug(response)
if response is None:
response = flask.Response(status=204)
return response
@app.route("/plot_get_dx_spots_per_month", methods=["POST"])
@csrf.exempt
def get_dx_spots_per_month():
response = flask.Response(json.dumps(bar_graph_spm.get_data()))
logger.debug(response)
if response is None:
response = flask.Response(status=204)
return response
@app.route("/plot_get_dx_spots_trend", methods=["POST"])
@csrf.exempt
def get_dx_spots_trend():
response = flask.Response(json.dumps(line_graph_st.get_data()))
logger.debug(response)
if response is None:
response = flask.Response(status=204)
return response
@app.route("/plot_get_hour_band", methods=["POST"])
@csrf.exempt
def get_dx_hour_band():
response = flask.Response(json.dumps(bubble_graph_hb.get_data()))
logger.debug(response)
if response is None:
response = flask.Response(status=204)
return response
@app.route("/plot_get_world_dx_spots_live", methods=["POST"])
@csrf.exempt
def get_world_dx_spots_live():
response = flask.Response(json.dumps(geo_graph_wdsl.get_data()))
logger.debug(response)
if response is None:
response = flask.Response(status=204)
return response
@app.route("/csp-reports", methods=['POST'])
@csrf.exempt
def csp_reports():
report_data = request.get_data(as_text=True)
logger.warning("CSP Report:")
logger.warning(report_data)
response=flask.Response(status=204)
return response
@app.context_processor
def inject_template_scope():
injections = dict()
def cookies_check():
value = request.cookies.get("cookie_consent")
return value == "true"
injections.update(cookies_check=cookies_check)
return injections
@app.after_request
def add_security_headers(resp):
resp.headers["Strict-Transport-Security"] = "max-age=1000"
resp.headers["X-Xss-Protection"] = "1; mode=block"
resp.headers["X-Frame-Options"] = "SAMEORIGIN"
resp.headers["X-Content-Type-Options"] = "nosniff"
resp.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
resp.headers["Cache-Control"] = "public, no-cache"
resp.headers["Pragma"] = "no-cache"
resp.headers["ETag"] = app.config["VERSION"]
#resp.headers["Report-To"] = '{"group":"csp-endpoint", "max_age":10886400, "endpoints":[{"url":"/csp-reports"}]}'
resp.headers["Content-Security-Policy"] = "\
default-src 'self';\
script-src 'self' cdnjs.cloudflare.com cdn.jsdelivr.net 'nonce-"+inline_script_nonce+"';\
style-src 'self' cdnjs.cloudflare.com cdn.jsdelivr.net;\
object-src 'none';base-uri 'self';\
connect-src 'self' cdn.jsdelivr.net cdnjs.cloudflare.com sidc.be;\
font-src 'self' cdn.jsdelivr.net;\
frame-src 'self';\
frame-ancestors 'none';\
form-action 'none';\
img-src 'self' data: cdnjs.cloudflare.com sidc.be;\
manifest-src 'self';\
media-src 'self';\
worker-src 'self';\
report-uri /csp-reports;\
"
return resp
#report-to csp-endpoint;\
#script-src 'self' cdnjs.cloudflare.com cdn.jsdelivr.net 'nonce-sedfGFG32xs';\
#script-src 'self' cdnjs.cloudflare.com cdn.jsdelivr.net 'nonce-"+inline_script_nonce+"';\
if __name__ == "__main__":
app.run(host="0.0.0.0")