Changeset 1362

Timestamp:

Sep 12, 2010, 2:39:47 AM (10 years ago)

Author:

teymour

Message:

Ajout du contenu des documents
+ correction de compute_latest.sh

Location:

cpc/trunk/project/batch/documents

Files:

• compute_latest.sh (modified) (1 diff)
• parse_metas.pl (modified) (2 diffs)

cpc/trunk/project/batch/documents/compute_latest.sh

@@ -1 +1 @@
 #!/bin/bash
+
+if ! test -e out/ ; then
+        mkdir out
+fi
 
 for file in `perl download_docs.pl`; do
   echo $file
-  file2=`echo $file | sed 's/\(pjl|ppl|ppr|rap|ta\)/out/'`
-  perl parse_document.pl $file > $file2
+  file2=`echo $file | sed 's/^\(pjl\|ppl\|ppr\|rap\|ta\)\//out\//'`
+  perl parse_metas.pl $file > $file2
 done;
 

cpc/trunk/project/batch/documents/parse_metas.pl

@@ -36 +36 @@
 $string =~ s/\s+/ /g;
 $string =~ s/"//g;
-if ($dir == "rap") {
-  $string =~ s/L\W\W([AÀEÈÉÊIOUYH])/L'\1/g;
-  $string =~ s/D\W\W([AÀEÈÉÊIOUYH])/D'\1/g;
-}
+
 $keywords = "";
 #print $string."\n";
@@ -159 +156 @@
 $categorie =~ s/Ç/ç/g;
 
+$string =~ s/<[^>]*>//gi;
+$string =~ s/"//gi;
+$string =~ s/[\n\t]/ /gi;
+$string =~ s/  +/ /gi;
 
+$string =~ s/if \(window!= top\) top\.location\.href=location\.href//i;
+$string =~ s/Recherche \| Aide \| Plan du site Accueil \&gt\; Documents parlementaires \&gt\; Les rapports législatifs//i;
+$string =~ s/_____ ASSEMBL'E NATIONALE CONSTITUTION DU 4 OCTOBRE 1958 TREIZIÈME LÉGISLATURE//i;
+$string =~ s/__*//i;
 #print "\n";
-print '{"source": "'.$source.'", "legislature": "'.$legislature.'", "id": "'.$id.'", "numero": "'.$num.'", "annexe": "'.$annexe.'", "date_depot": "'.$date0.'", "date_publi": "'.$date1.'", "auteurs": "'.$auteurs.'", "dossier": "'.$dossier.'", "type": "'.$type0.'", "type_details": "'.$type1.'", "titre": "'.$titre.'", "categorie": "'.$categorie.'", "motscles": "'.$keywords.'"}'."\n";
-
-
+print '{"source": "'.$source.'", "legislature": "'.$legislature.'", "id": "'.$id.'", "numero": "'.$num.'", "annexe": "'.$annexe.'", "date_depot": "'.$date0.'", "date_publi": "'.$date1.'", "auteurs": "'.$auteurs.'", "dossier": "'.$dossier.'", "type": "'.$type0.'", "type_details": "'.$type1.'", "titre": "'.$titre.'", "categorie": "'.$categorie.'", "motscles": "'.$keywords.'", "contenu": "'.$string.'"}'."\n";