#!/usr/bin/perl -w

# Récupérer https://www.gutenberg.org/cache/epub/feeds/rdf-files.tar.bz2 et le décompresser dans le répertoire courant

# pré-requis :
# cpan[1]> install XML::Fast



use HTTP::Tiny;
use strict;
use Encode qw(decode encode from_to);
use File::Find;
use XML::Fast;
use Data::Dumper;
use Cwd;

#use re 'debug';

my $basedir='';

my $from;
my $vu;

if (@ARGV)
{
	$from=$ARGV[0];
}

open LOADER,'>:utf8', 'textes.sql';
print LOADER <<"EOF";
DROP DATABASE textes;
CREATE DATABASE textes;
\\c textes
\\set ON_ERROR_STOP 1
BEGIN;
CREATE TABLE textes (livre text, ligne int, contenu text);
CREATE TEMP TABLE raw (contenu text);
EOF


find (\&indexe, './cache');

sub indexe
{
	return unless (-f  $_);
	return unless ( $_ =~ /\.rdf$/);
	my $document= do
	{
		local $/=undef;
		open (my $fh, "<:utf8", $_) or die "Cannot open $_";
		<$fh>;
	};
	my $content=xml2hash($document) or return;
	# Ignorer si pas français
	return unless (defined $content->{'rdf:RDF'}->{'pgterms:ebook'}->{'dcterms:language'} and ref($content->{'rdf:RDF'}->{'pgterms:ebook'}->{'dcterms:language'}) eq 'HASH');
	my $lang=$content->{'rdf:RDF'}->{'pgterms:ebook'}->{'dcterms:language'}->{'rdf:Description'}->{'rdf:value'}->{'#text'};
	return unless ($lang eq 'fr');

	# Alors ils sont complètement cons, ils peuvent avoir mis un tableau de {'dcterms:creator'}. Dans ce cas, je zappe (autre chose à foutre) FIXME
	return unless (defined $content->{'rdf:RDF'}->{'pgterms:ebook'}->{'dcterms:creator'} and ref($content->{'rdf:RDF'}->{'pgterms:ebook'}->{'dcterms:creator'}) eq 'HASH');

	my $author=$content->{'rdf:RDF'}->{'pgterms:ebook'}->{'dcterms:creator'}->{'pgterms:agent'}->{'pgterms:name'};
	my $id=$content->{'rdf:RDF'}->{'pgterms:ebook'}->{'-rdf:about'};

	$id =~ s/^ebooks\///; # On vire le ebooks du début de la reference

	my $title=$content->{'rdf:RDF'}->{'pgterms:ebook'}->{'dcterms:title'};

	$title =~ s/\r//g;
	$title =~ s/\n/ /g;
	print "$id - $title - $author - $lang\n";



	my @splitid=split('',$id);
	pop (@splitid); # Le dernier chiffre ne sert à rien
	my $splitid=join('/',@splitid) . '/' . $id;
	my $response = HTTP::Tiny->new->get("http://ftp.sunet.se/pub/etext/gutenberg/$splitid");
	unless ($response->{content}=~ /href="($id-\d+.txt)"/)
	{
		print "pas de contenu texte pour $title-$author.txt (http://ftp.sunet.se/pub/etext/gutenberg/$splitid)\n";
		return;
	}
	my $file=$1;
	print "Nouveau fichier=======\n";
	my $url="http://ftp.sunet.se/pub/etext/gutenberg/$splitid/$file";
	print "URL:$url\n";
	$response = HTTP::Tiny->new->get($url);
	my $data=$response->{content};
	unless ($data =~ /Character set encoding: (.*)$/m)
	{
		print "Cannot identify encoding\n";
		return;
	}
	my $encoding=$1;
	if ($encoding =~ /Unicode UTF-8|UTF-0/)# Y a des bouquins comme ça
	{
		$encoding = 'utf-8';
	}
	print "$encoding\n";
	print "$id -> $title-$author.txt\n";
	from_to($data,$encoding,'utf-8');
	# Rentre la chaîne compatible: remplacer les tabulations, remplacer les \
	$data =~ s/\\/\\\\/g;
	$data =~ s/\t/\\\t/g;
	$data =~ s/\r//g; # Parasites dans certains fichiers
	if (open OUT,">$basedir/books/$id.txt")
	{
		print OUT $data;
		close OUT;
	}
	else
	{
		print "Cannot open $basedir/books/$id.txt\n";
		return;
	}
	# Ok, importer le reste. On convertit les chaines en versions SQL
	$title =~ s/'/''/g;
	$author =~ s/'/''/g;
	# On vérifie que le fichier existe
	return unless (-e "$basedir/books/$id.txt");
	# Génération du SQL
	print LOADER "TRUNCATE raw;\n";
	print LOADER "\\copy raw from '$basedir/books/$id.txt'\n";
	print LOADER "INSERT INTO textes (livre,ligne,contenu) SELECT '$title, $author',row_number() over (),contenu FROM raw;\n";
	print LOADER "\n";
}
print LOADER "COMMIT;\n";
close LOADER;
