mirror of
git://scm.dxcluster.org/scm/spider
synced 2024-09-21 07:47:10 +00:00
add convert-users-v3-to-v4.pl
This commit is contained in:
Dirk Koopman
parent
7601606b4f
commit
df50cc60aa
8
Changes
8
Changes
@ -1,3 +1,11 @@
|
||||
20May20=======================================================================
|
||||
1. Backport convert-users-v3-to-v4.pl to allow creation of the new json
|
||||
formatted *text* based user file from the old v3 DB_File and Storable
|
||||
version. This can be done either online (with the node running) or offline
|
||||
with the node stopped. This is in preparation for the next update of the
|
||||
mojo branch. Running this program just *BEFORE* doing your next update of
|
||||
the mojo branch *should* ensure a seemless transition to the Storable &
|
||||
DB_File free version of the users file.
|
||||
24Apr20=======================================================================
|
||||
1. Add CTY-3008 prefixes
|
||||
22Apr20=======================================================================
|
||||
|
||||
14
UPGRADE.mojo
14
UPGRADE.mojo
@ -16,6 +16,9 @@ DXSpider node. But serious work on this won't start until we have a stable base
|
||||
on. Apart from anything else there will, almost certainly, need to be some internal data
|
||||
structure reorganisation before a decent web frontend could be constructed.
|
||||
|
||||
*IMPORTANT* There is an action needed to go from mojo build 228 and below. See items marked
|
||||
*IMPORTANT* below.
|
||||
|
||||
Upgrading is not for the faint of heart. There is no installation script (but there
|
||||
will be) so, for the time being, you need to do some manual editing. Also, while there is
|
||||
a backward path, it will involve moving various files from their new home (/spider/local_data),
|
||||
@ -53,7 +56,7 @@ You will need the following CPAN packages:
|
||||
If you are on a Debian based system (Devuan, Ubuntu, Mint etc) that is reasonably new (I use Ubuntu
|
||||
18.04 and Debian 10) then you can simply do:
|
||||
|
||||
sudo apt-get install libev-perl libmojolicious-perl libjson-perl libjson-xs-perl
|
||||
sudo apt-get install libev-perl libmojolicious-perl libjson-perl libjson-xs-perl libdata-structure-util-perl
|
||||
|
||||
or on Redhat based systems you can install the very similarly (but not the same) named
|
||||
packages. I don't the exact names but using anything less than Centos 7 is likely to cause
|
||||
@ -61,10 +64,11 @@ You will need the following CPAN packages:
|
||||
|
||||
If in doubt or it is taking too long to find the packages you should build from CPAN. Note: you may
|
||||
need to install the essential packages to build some of these. At the very least you will need
|
||||
to install 'make' (sudo apt-get install make) or just get everything you are likely to need with
|
||||
to install 'make' (sudo apt-get install make) or just get everything you are likely to need with:
|
||||
|
||||
sudo apt-get install build-essential.
|
||||
|
||||
sudo cpanm EV Mojolicious JSON JSON::XS
|
||||
sudo cpanm EV Mojolicious JSON JSON::XS Data::Structure::Util
|
||||
|
||||
# just in case it's missing
|
||||
sudo apt-get install top
|
||||
@ -162,6 +166,10 @@ if you have not already done this:
|
||||
sudo ln -s /spider/perl/console.pl /usr/local/bin/dx
|
||||
sudo ln -s /spider/perl/*dbg /usr/local/bin
|
||||
|
||||
*IMPORTANT* (for any build of dxspider) regardless of branch below build 229 run:
|
||||
|
||||
/spider/perl/convert-users-v3-to-v4.pl
|
||||
|
||||
Now in another window run:
|
||||
|
||||
watchdbg
|
||||
|
||||
143
perl/convert-users-v3-to-v4.pl
Executable file
143
perl/convert-users-v3-to-v4.pl
Executable file
@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env perl
|
||||
#
|
||||
# Convert users.v2 or .v3 to JSON .v4 format
|
||||
#
|
||||
# It is believed that this can be run at any time...
|
||||
#
|
||||
# Copyright (c) 2020 Dirk Koopman G1TLH
|
||||
#
|
||||
#
|
||||
#
|
||||
|
||||
# make sure that modules are searched in the order local then perl
|
||||
|
||||
BEGIN {
|
||||
# root of directory tree for this system
|
||||
$root = "/spider";
|
||||
$root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'};
|
||||
unshift @INC, "$root/perl"; # this IS the right way round!
|
||||
unshift @INC, "$root/local";
|
||||
}
|
||||
|
||||
use strict;
|
||||
|
||||
use SysVar;
|
||||
use DXUser;
|
||||
use DXUtil;
|
||||
use JSON;
|
||||
use Data::Structure::Util qw(unbless);
|
||||
use Time::HiRes qw(gettimeofday tv_interval);
|
||||
use IO::File;
|
||||
use File::Copy;
|
||||
use Carp;
|
||||
use DB_File;
|
||||
|
||||
use 5.10.1;
|
||||
|
||||
my $ufn;
|
||||
my $fn = "users";
|
||||
|
||||
my $json = JSON->new()->canonical(1);
|
||||
my $ofn = localdata("$fn.v4");
|
||||
my $convert;
|
||||
|
||||
eval {
|
||||
require Storable;
|
||||
};
|
||||
|
||||
if ($@) {
|
||||
if ( ! -e localdata("$fn.v3") && -e localdata("$fn.v2") ) {
|
||||
$convert = 2;
|
||||
}
|
||||
LogDbg('',"the module Storable appears to be missing!!");
|
||||
LogDbg('',"trying to continue in compatibility mode (this may fail)");
|
||||
LogDbg('',"please install Storable from CPAN as soon as possible");
|
||||
}
|
||||
else {
|
||||
import Storable qw(nfreeze thaw);
|
||||
$convert = 3 if -e localdata("users.v3") && !-e $ufn;
|
||||
}
|
||||
|
||||
die "need to have a $fn.v2 or (preferably) a $fn.v3 file in /spider/data or /spider/local_data\n" unless $convert;
|
||||
|
||||
if (-e $ofn || -e "$ofn.n") {
|
||||
my $nfn = localdata("$fn.v4.json");
|
||||
say "You appear to have (or are using) $ofn, creating $nfn instead";
|
||||
$ofn = $nfn;
|
||||
} else {
|
||||
$ofn = "$ofn.n";
|
||||
say "using $ofn.n for output";
|
||||
}
|
||||
|
||||
|
||||
# do a conversion if required
|
||||
if ($convert) {
|
||||
my ($key, $val, $action, $count, $err) = ('','',0,0,0);
|
||||
my $ta = [gettimeofday];
|
||||
my $ofh = IO::File->new(">$ofn") or die "cannot open $ofn ($!)\n";
|
||||
|
||||
my %oldu;
|
||||
LogDbg('',"Converting the User File from V$convert to $fn.v4 ");
|
||||
LogDbg('',"This will take a while, maybe as much as 10 secs");
|
||||
my $odbm = tie (%oldu, 'DB_File', localdata("users.v$convert"), O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn.v$convert ($!) [rebuild it from user_asc?]";
|
||||
for ($action = R_FIRST; !$odbm->seq($key, $val, $action); $action = R_NEXT) {
|
||||
my $ref;
|
||||
if ($convert == 3) {
|
||||
eval { $ref = storable_decode($val) };
|
||||
}
|
||||
else {
|
||||
eval { $ref = asc_decode($val) };
|
||||
}
|
||||
unless ($@) {
|
||||
if ($ref) {
|
||||
unbless $ref;
|
||||
$ofh->print("$ref->{call}\t" . $json->encode($ref) . "\n");
|
||||
$count++;
|
||||
}
|
||||
else {
|
||||
$err++
|
||||
}
|
||||
}
|
||||
else {
|
||||
Log('err', "DXUser: error decoding $@");
|
||||
}
|
||||
}
|
||||
undef $odbm;
|
||||
untie %oldu;
|
||||
my $t = _diffms($ta);
|
||||
LogDbg('',"Conversion from users.v$convert to $ofn completed $count records $err errors $t mS");
|
||||
$ofh->close;
|
||||
}
|
||||
|
||||
exit 0;
|
||||
|
||||
sub asc_decode
|
||||
{
|
||||
my $s = shift;
|
||||
my $ref;
|
||||
$s =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg;
|
||||
eval '$ref = ' . $s;
|
||||
if ($@) {
|
||||
LogDbg('err', "DXUser::asc_decode: on '$s' $@");
|
||||
$ref = undef;
|
||||
}
|
||||
return $ref;
|
||||
}
|
||||
|
||||
sub storable_decode
|
||||
{
|
||||
my $ref;
|
||||
$ref = thaw(shift);
|
||||
return $ref;
|
||||
}
|
||||
|
||||
sub LogDbg
|
||||
{
|
||||
my (undef, $s) = @_;
|
||||
say $s;
|
||||
}
|
||||
|
||||
sub Log
|
||||
{
|
||||
say shift;
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user