visit counter & numer of users

2024-09-21 07:27:09 +00:00 · 2024-03-24 07:47:27 +01:00 · 2024-03-24 07:47:27 +01:00 · b6fabbdb92
commit b6fabbdb92
parent 1592993225
14 changed files with 263 additions and 8657 deletions
--- a/CITATION.cff
+++ b/CITATION.cff
@ -25,4 +25,4 @@ keywords:
  - spiderweb
 license: GPL-3.0
 version: v2.5.3
-date-released: 2024-03-10
+date-released: 2024-03-24
--- a/README.md
+++ b/README.md
@ -13,7 +13,7 @@
 - **Author:** Corrado Gerbaldo - [IU1BOW](https://www.qrz.com/db/IU1BOW)
 - **Mail:** <corrado.gerbaldo@gmail.com>
 - **Licensing:** Gpl V3.0 see [LICENSE](LICENSE) file.
- **Languages:** This application is written in Python 3.11/flask,Javascript and HTML
+- **Languages:** This application is written in Python 3.12/flask,Javascript and HTML

 ___
 **DXSpider** is a great DX Cluster software that has useful telnet interface. 
@ -73,12 +73,6 @@ foo@bar:~$ pip install flask
 foo@bar:~$ pip install Flask-minify
 foo@bar:~$ pip install flask_wtf
 foo@bar:~$ pip install pandas
-```
-Then you have to install mysql libraries**:
-```console
-foo@bar:~$ pip install mysql-connector-python
-foo@bar:~$ pip install --upgrade mysql-connector-python==8.0.12
-
 ```

 ### Configuration  
--- a/data/.gitignore
+++ b/data/.gitignore
@ -0,0 +1,2 @@
+*
+!.gitignore
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@ -1,11 +1,22 @@
 ### Change log
+Date: 24/03/2024 
+Release: v2.5.3
+- tested with Python 3.12
+- replaced mysql driver with mariadb driver
+- upgraded Echarts lib from 5.3 to 5.5
+- upgraded flag-icon-css lib from to 7.2
+- upgraded bootstrap to 5.3.3
+- issue [#51](https://github.com/coulisse/spiderweb/issues/51): added total number of users & nodes connected Issue
+- issue [#56](https://github.com/coulisse/spiderweb/issues/56): added a simple counter
+  
+___
 Date: 10/03/2024 
 Release: v2.5.3
 - adapted card size and text for mobile
 - removed monitor
 - removed cookie consent banner, since this application uses only technical cookies
- issue [#51] (https://github.com/coulisse/spiderweb/issues/51)   -- just for caching
- security [#22] (https://github.com/coulisse/spiderweb/security/dependabot/22)
+- issue [#51](https://github.com/coulisse/spiderweb/issues/51)   -- just for caching
+- security issue [#22](https://github.com/coulisse/spiderweb/security/dependabot/22)
  
 ___
 Date: 03/12/2023 
--- a/lib/cty.py
+++ b/lib/cty.py
@ -17,7 +17,7 @@ logging.basicConfig(
 )
 # TODO: url from conf parameter
 url = "https://www.country-files.com/cty/cty_wt_mod.dat"
-cty_local = os.path.dirname(__file__) + "/../static/data/cty_wt_mod.dat"
+cty_local = os.path.dirname(__file__) + "/../data/cty_wt_mod.dat"
 country_file = os.path.dirname(__file__) + "/../cfg/country.json"
 # -------------------------------------------------------------------------------------
 #  download country files cty.dat
@ -166,6 +166,7 @@ def parse_alias(alias, master):
 #  load file from configuration, containing all world country, with related ISO codes
 # -------------------------------------------------------------------------------------
 def load_country():
+    logging.info('loading:' +country_file)
    with open(country_file) as json_country:
        return json.load(json_country)

--- a/lib/qry.py
+++ b/lib/qry.py
@ -1,10 +1,9 @@
 # *****************************************************************************************
-# module used to make query to mysql
+# module used to make query to mariadb
 # TODO: manage polymorfism and use only one qry sign
 # *****************************************************************************************
 # import MySQLdb as my
-import mysql.connector as my
-from mysql.connector import pooling
+import mariadb as my
 import logging
 import json
 import pandas as pd
@ -34,18 +33,25 @@ class query_manager:
                return

        logging.info("config file loaded")
-        self.__cnxpool = pooling.MySQLConnectionPool(
+
+
+        self.__cnxpool = my.ConnectionPool(
            host=cfg["mysql"]["host"],
+            #port=3306,
            user=cfg["mysql"]["user"],
            passwd=cfg["mysql"]["passwd"],
            db=cfg["mysql"]["db"],
-            charset="latin1",
-            #                    charset='utf8mb4',
-            #                    collation = 'utf8mb4_general_ci',
+#            charset="latin1",
            pool_name="spider_pool",
-            use_pure=True,
+#            use_pure=True,
            pool_size=3,
+            pool_validation_interval=250
        )
+
+
+
+
+
        logging.info("db connection pool created")

    # normal query
--- a/requirements.txt
+++ b/requirements.txt
@ -1,41 +1,31 @@
-astroid==2.12.14
-blinker==1.6.2
-charset-normalizer==2.1.1
-click==8.1.3
-dill==0.3.6
-docopt-ng==0.8.1
-easywatch==0.0.5
-Flask==2.3.3
-Flask-Consent==0.0.3
-Flask-Minify==0.41
-Flask-WTF==1.1.1
+blinker==1.7.0
+charset-normalizer==3.3.2
+click==8.1.7
+Flask==3.0.2
+Flask-Minify==0.42
+Flask-WTF==1.2.1
 htmlmin==0.1.12
-idna==3.4
-isort==5.11.4
+idna==3.6
 itsdangerous==2.1.2
 Jinja2==3.1.3
 jsmin==3.0.1
-lazy-object-proxy==1.9.0
 lesscpy==0.15.1
-markup==0.2
-MarkupSafe==2.1.1
-mccabe==0.7.0
-mysql-connector-python>=8.2.0
-numpy==1.24.1
-pandas==1.5.2
-platformdirs==2.6.2
+mariadb==1.1.10
+MarkupSafe==2.1.5
+numpy==1.26.4
+packaging==24.0
+pandas==2.2.1
 ply==3.11
-protobuf==4.21.12
-python-dateutil==2.8.2
-pytz==2022.7
-rcssmin==1.1.1
+python-dateutil==2.9.0.post0
+pytz==2024.1
+rcssmin==1.1.2
 requests==2.31.0
+setuptools==68.2.2
 six==1.16.0
-tomlkit==0.11.6
-urllib3==2.0.7
-watchdog==3.0.0
-Werkzeug==2.3.8
-wrapt==1.14.1
-WTForms==3.0.1
+tzdata==2024.1
+urllib3==2.2.1
+Werkzeug==3.0.1
+wheel==0.41.2
+WTForms==3.1.2
 xmltodict==0.13.0
-xxhash==3.1.0
+xxhash==3.4.1
--- a/scripts/build.sh
+++ b/scripts/build.sh
@ -134,9 +134,9 @@ if [ "$1" == "-r" ]; then
 	sed -i '/staticjinja==/d' ../requirements.txt
 	sed -i '/lighthouse==/d' ../requirements.txt

-	echo 'force some requirements...'
-	sed -i 's/mysql-connector-python==8.0.31/mysql-connector-python>=8.0.31/' ../requirements.txt
-	sed -i 's/mysql-connector-python==8.2.0/mysql-connector-python>=8.2.0/' ../requirements.txt
+	#echo 'force some requirements...'
+	#sed -i 's/mysql-connector-python==8.0.31/mysql-connector-python>=8.0.31/' ../requirements.txt
+	#sed -i 's/mysql-connector-python==8.2.0/mysql-connector-python>=8.2.0/' ../requirements.txt

 	if ! sed -i '7,25s/level=DEBUG/level=INFO/g' ${app_ini}; then               
 		echo 'ERROR settimg loglevel=INFO '
--- a/scripts/dxcluster_schema_for_test.sh
+++ b/scripts/dxcluster_schema_for_test.sh
@ -9,7 +9,7 @@ chr() {
 }

 db_insert () {
-    n=10000
+    n=2000000
    for (( i=1; i<=${n}; i++ ))
    do
        freq=$(shuf -i 100-50000 -n 1)
@ -25,8 +25,8 @@ db_insert () {
        #timestamp=$(shuf -i 1673759569-1673763169 -n 1)
        #epoch_start=$((${curr_epoch_time}-3600*24*365*2))
        epoch_start=$((${curr_epoch_time}-3600))
-        echo ${curr_epoch_time}
-        echo ${epoch_start}
+        #echo ${curr_epoch_time}
+        #echo ${epoch_start}
        timestamp=$(shuf -i ${epoch_start}-${curr_epoch_time} -n 1)

        cs_letter_1=$(chr $(shuf -i 65-90 -n1))
@ -43,10 +43,10 @@ db_insert () {
        #sudo mysql -uroot dxcluster -e "INSERT INTO spot VALUES (${i},${freq},'${callsign}',UNIX_TIMESTAMP(),'DUMMY TEST','IU1BOW',${spotdxcc},${spotterdxcc},'IU1BOW-2',${spotitu},${spotcq},${spotteritu},${spottercq},NULL,NULL,'5.198.229.129');"
        sleep 3
        p=$(( ${i}*100/${n} ))
-        echo -ne ${p}'% \r'
+      #  echo -ne ${p}'% \r'
    done

-    echo -ne '\n'
+   # echo -ne '\n'
 }


--- a/scripts/mysql2sqlite.sh
+++ b/scripts/mysql2sqlite.sh
@ -0,0 +1,126 @@
+
+#!/bin/bash
+echo this script will convert your mysql db to sqllite databases
+
+
+#TODO: 
+#
+# read dxvars
+# check sqllite perl
+# dump mysql
+# create table in sqlite
+# import in sqlite
+# create indexes
+# change dxvars.pm
+# 
+
+sqlite_db=dxcluster.db 
+mysql_dump_db=$(mktemp) 
+mysql_dump_db="mysql.sql"   #TODO: remove
+
+progress_bar() {
+    local width=50
+    local percent="$1"
+    local filled_width=$((width * percent / 100))
+    local dots="$(printf '%*s' "$filled_width" | tr ' ' '=')"
+    local spaces="$(printf '%*s' "$((width - filled_width))" | tr ' ' ' ')"
+    echo -ne "[$dots$spaces] ($percent%)\r"
+}
+
+
+#Empty database
+if ! >  ${sqlite_db}; 
+  then
+    echo 'Error empting sqlite db: ' ${sqlite_db}
+  exit 1
+else
+    echo 'sqlite db created: ' ${sqlite_db}
+fi
+
+#dump mysql data
+#TODO: remove comments
+#read -p 'MySQL User: ' user
+#
+#if ! mysqldump -u ${user} -p --skip-create-options  --compatible=ansi --skip-extended-insert  --compact --single-transaction  --databases dxcluster \
+#  | grep "INSERT INTO" \
+#  | sed -e ':a' -e 'N' -e '$!ba' -e 's/,\n)/\n)/'\
+#  | sed -e 's/\\'\''/'\'''\''/g'\
+#  > ${mysql_dump_db}; 
+#  then
+#	    echo 'Error on dumping mysql data'
+#        exit 1
+#  else
+#        echo 'dump created: ' ${mysql_dump_db}
+#fi		
+
+#create table spot
+if ! sqlite3 ${sqlite_db} <<EOF
+CREATE TABLE "spot" (
+  "rowid" INTEGER PRIMARY KEY,
+  "freq" REAL NOT NULL,
+  "spotcall" TEXT NOT NULL,
+  "time" INTEGER NOT NULL,
+  "comment" TEXT DEFAULT NULL,
+  "spotter" TEXT NOT NULL,
+  "spotdxcc" INTEGER DEFAULT NULL,
+  "spotterdxcc" INTEGER DEFAULT NULL,
+  "origin" TEXT DEFAULT NULL,
+  "spotitu" INTEGER DEFAULT NULL,
+  "spotcq" INTEGER DEFAULT NULL,
+  "spotteritu" INTEGER DEFAULT NULL,
+  "spottercq" INTEGER DEFAULT NULL,
+  "spotstate" TEXT DEFAULT NULL,
+  "spotterstate" TEXT DEFAULT NULL,
+  "ipaddr" TEXT DEFAULT NULL
+);
+EOF
+  then
+    echo 'Error on creating table spot in Sqlite'
+    exit 1
+  else
+    echo 'Table spot created in sqlite db: ' ${sqlite_db}
+fi
+
+#import spot in sqlite
+max_insert=$(wc -l ${mysql_dump_db}|cut -d ' ' -f1)
+echo 'Importing dump into Sqlite' ${max_insert} 'rows: '
+
+counter=0
+sv_perc=-1
+while IFS= read -r line; do
+    let "counter++"
+    if ! sqlite3 ${sqlite_db} "${line}";
+      then 
+        echo '...at line: ' ${counter} ' | ' ${line}
+    fi
+    perc=$((   ${counter} * 100 / ${max_insert}  ))
+    if [ ${perc} -ne ${sv_perc} ]; then
+      sv_perc=${perc}
+      progress_bar ${perc}
+    fi
+done < ${mysql_dump_db}
+
+echo 'Sqlite db imported: ' ${sqlite_db}
+
+
+#create index 
+echo 'Creating indexes...'
+if ! sqlite3 ${sqlite_db} <<EOF
+CREATE INDEX idx_spot_spotcall ON spot (spotcall);
+CREATE INDEX idx_spot_spotter ON spot (spotter);
+EOF
+  then
+    echo 'Error on creating indexes on spot in Sqlite'
+    exit 1
+  else
+    echo 'Indexes created in sqlite db: ' ${sqlite_db}
+fi
+
+
+
+exit  #TODO: remove exit
+#remove dump file
+rm ${mysql_dump_db}; 
+
+echo done
+
--- a/static/data/cty_wt_mod.dat
+++ b/static/data/cty_wt_mod.dat
--- a/templates/_base.html
+++ b/templates/_base.html
@ -16,10 +16,10 @@
 	<link rel="manifest" href="/static/pwa/manifest.webmanifest">
 	<link rel="stylesheet" href="/static/css/rel/style.min.css">
 	<!-- Bootstrap CSS -->
-	<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.3/dist/css/bootstrap.min.css"
-		integrity="sha384-rbsA2VBKQhggwzxH7pPCaAqO46MgnOM80zW1RWuH61DGLwZJEdK2Kadq2F9CUG65" crossorigin="anonymous">
+	<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.3.3/css/bootstrap.min.css"
+		integrity="sha512-jnSuA4Ss2PkkikSOLtYs8BlYIeeIK1h99ty4YfvRPAlzr377vr3CXDb7sb7eEEBYjDtcYj+AjBH3FLv5uSJuXg==" crossorigin="anonymous">
 	<!-- Flag Icon CSS -->		
-	<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/flag-icon-css/6.15.0/css/flag-icons.min.css"
+	<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/flag-icon-css/7.2.0/css/flag-icons.min.css"
 		integrity="sha512-bZBu2H0+FGFz/stDN/L0k8J0G8qVsAL0ht1qg5kTwtAheiXwiRKyCq1frwfbSFSJN3jooR5kauE0YjtPzhZtJQ=="
 		crossorigin="anonymous" referrerpolicy="no-referrer" />
 	<!--  Tom-Select CSS -->
@ -91,11 +91,21 @@
 		{% block contents %}
 		{% endblock contents %}
 	</div>
+
 	<footer class="page-footer font-small blue">
+
+		<hr class="hr" />	
+		<div class="text-center ">
+			<span class="bi-person-up" role="button" aria-label="funnel-fill"></span>			
+			Website unique visits: <strong>{{ visits }}</strong>
+		</div>		
+				
 		<div class="footer-copyright text-center py-3">
 			<span class="copyleft">&copy;</span> Copyleft:
 			<span id="copyDate"></span>
+			&nbsp;
 			<a href="https://github.com/coulisse/spiderweb/" target="blank" rel="noopener">IU1BOW - Spiderweb</a>
+			&nbsp;
 			<span id="version">v2.5.3</span>
 		</div>					
 	</footer>
@ -108,15 +118,14 @@
 	<script defer src="static/js/rel/common.min.js"></script>

 	<!-- Bootstrap -->
-	<script defer src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.3/dist/js/bootstrap.bundle.min.js"
-		integrity="sha384-kenU1KFdBIe4zVF0s0G1M5b4hcpxyD9F7jL+jjXkk+Q2h455rYXK/7HAuoJl+0I4"
+	<script defer src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.3.3/js/bootstrap.bundle.min.js"
+		integrity="sha512-7Pi/otdlbbCR+LnW+F7PwFcSDJOuUJB3OxtEHbg4vSMvzvJjde4Po1v4BR9Gdc9aXNUNFVUY+SK51wWT8WF0Gg=="
 		crossorigin="anonymous"></script>

 	<!-- Tom-select library -->
 	<script defer src="https://cdn.jsdelivr.net/npm/tom-select@2.3.1/dist/js/tom-select.complete.min.js" 
 		integrity="sha384-cnROoUgVILyibe3J0zhzWoJ9p2WmdnK7j/BOTSWqVDbC1pVw2d+i6Q/1ESKJKCYf" 
 		crossorigin="anonymous"></script>
-
 	</body>
 	{% block app_scripts %}
 	<script async src="static/js/rel/callsign_search.min.js"></script>
--- a/templates/plots.html
+++ b/templates/plots.html
@ -49,7 +49,11 @@

 	<div class="container-fluid">
 		<div class="shadow-lg mb-5 bg-body rounded">
-			<strong>Physically connected callsigns to {{ mycallsign }}</strong>
+			<strong>{{ mycallsign }} telnet nodes & users online. </strong>
+			<br>
+				Nodes: <strong>{{ who|selectattr('type', 'equalto', 'NODE DXSP')|map(attribute='type')|map('upper')|list|length }}</strong> 
+			<br>
+				Users: <strong> {{ who|selectattr('type', 'equalto', 'USER EXT')|map(attribute='type')|map('upper')|list|length }}</strong> 
 			<hr>
 			<table class="table table-striped table-borderless table-sm text-responsive table-hover">
 				<thead id="telnet-thead">
@ -86,8 +90,8 @@ var band_frequencies={{bands["bands"]|tojson|safe}};

 {% block app_scripts %}
 {{ super() }}
-<script defer src="https://cdnjs.cloudflare.com/ajax/libs/echarts/5.4.3/echarts.min.js"
-integrity="sha512-EmNxF3E6bM0Xg1zvmkeYD3HDBeGxtsG92IxFt1myNZhXdCav9MzvuH/zNMBU1DmIPN6njrhX1VTbqdJxQ2wHDg=="
+<script defer src="https://cdnjs.cloudflare.com/ajax/libs/echarts/5.5.0/echarts.min.js"
+integrity="sha512-k37wQcV4v2h6jgYf5IUz1MoSKPpDs630XGSmCaCCOXxy2awgAWKHGZWr9nMyGgk3IOxA1NxdkN8r1JHgkUtMoQ=="
 crossorigin="anonymous" referrerpolicy="no-referrer"></script>
 <script defer src="static/js/rel/plot.min.js"></script>
 {% endblock app_scripts %}
--- a/webapp.py
+++ b/webapp.py
@ -48,6 +48,7 @@ else:
 app.jinja_env.trim_blocks = True
 app.jinja_env.lstrip_blocks = True    

+
 # load config file
 with open("cfg/config.json") as json_data_file:
    cfg = json.load(json_data_file)
@ -66,6 +67,31 @@ with open("cfg/modes.json") as json_modes:
 with open("cfg/continents.json") as json_continents:
    continents_cq = json.load(json_continents)

+#load visitour counter
+visits_file_path = "data/visits.json"
+try:
+    # Load the visits data from the file
+    with open(visits_file_path) as json_visitors:
+        visits = json.load(json_visitors)
+except FileNotFoundError:
+    # If the file does not exist, create an empty visits dictionary
+    visits = {}
+
+#save visits
+def save_visits():
+    with open(visits_file_path, "w") as json_file:
+        json.dump(visits, json_file)
+    logging.info('visit saved on: '+ visits_file_path)
+
+# saving scheduled
+def schedule_save():
+    save_visits()
+    threading.Timer(1000, schedule_save).start()
+
+# Start scheduling
+schedule_save()
+
+
 # read and set default for enabling cq filter
 if cfg.get("enable_cq_filter"):
    enable_cq_filter = cfg["enable_cq_filter"].upper()
@ -125,9 +151,11 @@ def get_adxo():
    adxo_events = get_adxo_events()
    threading.Timer(12 * 3600, get_adxo).start()

-
 get_adxo()

+
+
+
 # create data provider for charts
 heatmap_cbp = ContinentsBandsProvider(logger, qm, continents_cq, band_frequencies)
 bar_graph_spm = SpotsPerMounthProvider(logger, qm)
@ -160,9 +188,22 @@ def get_nonce():
    inline_script_nonce = secrets.token_hex()
    return inline_script_nonce

+#check if it is a unique visitor
+def visitor_count():
+    user_ip = request.remote_addr
+    if user_ip not in visits:
+        visits[user_ip] = 1
+    else:
+        visits[user_ip] += 1        
+    
+
@app.route("/", methods=["GET"])
@app.route("/index.html", methods=["GET"])
 def spots():
+    
+    
+    visitor_count();
+
    response = flask.Response(
        render_template(
            "index.html",
@ -171,6 +212,7 @@ def spots():
            telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
            mail=cfg["mail"],
            menu_list=cfg["menu"]["menu_list"],
+            visits=len(visits),            
            enable_cq_filter=enable_cq_filter,
            timer_interval=cfg["timer"]["interval"],
            adxo_events=adxo_events,
@ -211,6 +253,7 @@ def sw():
 def root():
    return app.send_static_file("html/offline.html")

+#used for plots
@app.route("/world.json")  
 def world_data():
    return app.send_static_file("data/world.json")
@ -226,6 +269,7 @@ def plots():
            telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
            mail=cfg["mail"],
            menu_list=cfg["menu"]["menu_list"],
+            visits=len(visits),                     
            who=whoj,
            continents=continents_cq,
            bands=band_frequencies,
@ -257,6 +301,7 @@ def propagation():
            telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
            mail=cfg["mail"],
            menu_list=cfg["menu"]["menu_list"],
+            visits=len(visits),                     
            solar_data=solar_data
        )
    )
@ -274,6 +319,7 @@ def cookies():
            telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
            mail=cfg["mail"],
            menu_list=cfg["menu"]["menu_list"],
+            visits=len(visits),                     
        )
    )
    return response
@ -288,6 +334,7 @@ def privacy():
            telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
            mail=cfg["mail"],
            menu_list=cfg["menu"]["menu_list"],
+            visits=len(visits),                     
        )
    )
    return response
@ -309,6 +356,7 @@ def callsign():
            telnet=cfg["telnet"]["host"]+":"+cfg["telnet"]["port"],
            mail=cfg["mail"],
            menu_list=cfg["menu"]["menu_list"],
+            visits=len(visits),                     
            timer_interval=cfg["timer"]["interval"],
            callsign=callsign,
            adxo_events=adxo_events,