Outils_archives/corpus_web
3
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Feat: ajout de l'extraction des liens internes et externes

Browse source
This commit is contained in:
cy_altern 2023-05-29 18:04:17 +02:00
parent 20663f23ff
commit 504e7affa4
3 changed files with 210 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
content/contenu_jsonl.html
View file

@ -45,6 +45,23 @@
<div class="aff_masq_">Titres :</div> <div class="aff_masq_">Titres :</div>
<div class="details_">[(#VALEUR|base64_decode|recup_titres)]</div> <div class="details_">[(#VALEUR|base64_decode|recup_titres)]</div>
</li> </li>
<li>
<div class="aff_masq_">Trackers :</div>
<div class="details_">[(#VALEUR|base64_decode|trouve_trackers)]</div>
</li>
<li>
<div class="aff_masq_">Rezos socios :</div>
<div class="details_">[(#VALEUR|base64_decode|trouve_rezos)]</div>
</li>
<li>
<div class="aff_masq_">Liens externes :</div>
<div class="details_">[(#VALEUR|base64_decode|recup_liens_ext)]</div>
</li>
<li>
<div class="aff_masq_">Liens internes :</div>
<div class="details_">[(#VALEUR|base64_decode|recup_liens_int)]</div>
</li>
] ]
[(#CLE|=={extractionContent}|oui) [(#CLE|=={extractionContent}|oui)
<BOUCLE_ss(DATA){source table, #VALEUR}> <BOUCLE_ss(DATA){source table, #VALEUR}>
@ -52,7 +69,7 @@
<li> <li>
<div class="aff_masq_">[(#CLE|replace{htmlmeta:head:, meta }|replace{boilerpipe:, ''}|ucfirst)] : </div> <div class="aff_masq_">[(#CLE|replace{htmlmeta:head:, meta }|replace{boilerpipe:, ''}|ucfirst)] : </div>
<div class="details_"> <div class="details_">
[(#VALEUR|print|replace{<br>,§§}|replace{<br />,§§}|replace{<,&lt;}|replace{>,&gt;}|replace{§§,<br>}|replace{"\n", <br>})] [(#VALEUR|print|supprimer_com|replace{<br>,§§}|replace{<br />,§§}|replace{<,&lt;}|replace{>,&gt;}|replace{§§,<br>}|replace{"\n", <br>})]
</div> </div>
</li> </li>
] ]

157
corpus_web_fonctions.php
View file

@ -191,4 +191,159 @@ function recup_aspi($nom_fichier_jsonl) {
$T = explode('_', $nom_fichier_jsonl)[1]; $T = explode('_', $nom_fichier_jsonl)[1];
$T = explode('-', $T); $T = explode('-', $T);
return $T[0].'-'.$T[1].'-'.$T[2]; return $T[0].'-'.$T[1].'-'.$T[2];
} }
function supprimer_com($boiler_text) {
return preg_replace('/Poster un nouveau commentaire.*/si', '', $boiler_text);
}
function trouve_trackers($html) {
// google(-?)analytics|xiti.com|ranktrackr.net
$trackers = [];
preg_match_all('/google(-?)analytics|xiti\.com|ranktrackr\.net|seoposition\.com/si', $html, $match, PREG_SET_ORDER);
if ($match) {
foreach($match as $m) {
if (strtolower(str_replace(['-', ' '], '', $m[0])) === 'googleanalytics') {
if (!in_array('google analytics', $trackers)) {
$trackers[] = 'google analytics';
}
} else {
if (!in_array(strtolower($m[0]), $trackers)) {
$trackers[] = strtolower($m[0]);
}
}
}
}
return count($trackers) ? "<ul><li>".join("</li><li>", $trackers)."</li></ul>" : '';
}
function trouve_rezos($html) {
// facebook.com|twitter.com|pinterest.com|plusone
$rezos = [];
preg_match_all('/facebook\.com|twitter\.com|plusone/si', $html, $match, PREG_SET_ORDER);
if ($match) {
foreach($match as $m) {
switch (strtolower($m[0])) {
case 'facebook.com':
if (!in_array('Facebook', $rezos)) {
$rezos[] = 'Facebook';
}
break;
case 'twitter.com':
if (!in_array('Twitter', $rezos)) {
$rezos[] = 'Twitter';
}
break;
case 'plusone':
if (!in_array('Google Plus One', $rezos)) {
$rezos[] = 'Google Plus One';
}
break;
}
}
}
return count($rezos) ? "<ul><li>".join("</li><li>", $rezos)."</li></ul>" : '';
}
function recup_liens_ext($html) {
// <a href="http(s?)://(?!www.o2zone.tv|t.co)([^"]*)
$liens_ext = [];
$exclus = [
'w3.org',
'adobe.com',
'xiti.com',
'ranktrackr.net',
'seoposition.com',
'facebook.com',
'twitter.com',
];
preg_match_all('/<a href="http(s?):\/\/(?!www\.o2zone.tv|t\.co)([^"]*)/si', $html, $match, PREG_SET_ORDER);
if ($match) {
foreach($match as $m) {
// domaines exclus
$domaine = explode('/', $m[2])[0];
if (substr_count($domaine, '.') > 1) {
$Tdomaine = explode('.', $domaine);
$domaine = array_pop($Tdomaine);
$domaine = array_pop($Tdomaine).'.'.$domaine;
}
if(in_array($domaine, $exclus)) {
continue;
}
if (!in_array($m[2], $liens_ext)) {
$liens_ext[] = $m[2];
}
}
}
return count($liens_ext) ? "<ul><li>".join("</li><li>", $liens_ext)."</li></ul>" : '';
}
function recup_liens_int($html) {
// <a href="http(s?)://(?!www.o2zone.tv|t.co)([^"]*)
$liens_int = [];
$txt_exclus = [
'Accueil',
'Edito',
'En savoir plus...',
'Plus de mots-clés',
'Lire la suite »',
' Lire la suite »',
'&nbsp;Lire&nbsp;la&nbsp;suite&nbsp;»',
'Ajouter un commentaire',
'dernier »',
'suivant ›',
'XHTML',
'répondre',
'Téléchargez le plus récent lecteur Flash',
'1',
'2',
'3',
'4',
'5',
'6',
'7',
'8',
'9',
'Partie1',
'Partie2',
'Partie3',
'Partie4',
];
$url_exclus = [
'/',
'/la-tele',
'/video',
'/la-boite-outils',
'/lassociation',
'/forum',
'/user',
'/aide-du-site-o2zone',
'/propos',
'/filter/tips',
'/rss.xml',
];
preg_match_all('/<a href="(http(s?):\/\/www\.o2zone\.tv)?(\/[^"]*)?([^>]*)?">(<(img|p|acronym)[^>]*>)?(.*?)<\/a>/si', $html, $match, PREG_SET_ORDER);
if ($match) {
foreach($match as $m) {
$m[7] = preg_replace('/<(img|(\/)?span)[^>]*>/si', '', $m[7]);
// liens exclus
if(in_array($m[3], $url_exclus) || str_contains($m[3], '#comment') || str_contains($m[3], '/user/') || in_array($m[7], $txt_exclus) || str_replace([' ', '&nbsp;'], '', $m[7]) === '') {
continue;
}
if (!array_key_exists($m[3], $liens_int)) {
$liens_int[$m[3]] = $m[7];
}
}
}
$l_int = [];
if (count($liens_int)) {
foreach ($liens_int as $url => $titre) {
$l_int[] = $url.' : '.$titre;
}
}
return count($l_int) ? "<ul><li>".join("</li><li>", $l_int)."</li></ul>" : '';
}

42
json_affiche_corpus.json.html
View file

@ -9,20 +9,36 @@
"title": "Meta title", "title": "Meta title",
"desc": "Meta description", "desc": "Meta description",
"keywords": "Meta keywords", "keywords": "Meta keywords",
"trackers": "Trackers",
"rezos": "Rézos socios",
"titres": "titres", "titres": "titres",
"liens_ext": "Liens externes",
"liens_int": "Liens internes",
"text": "Texte" "text": "Texte"
}, },
"filtreCol" : { "filtreCol" : {
"site" : "select", "site" : "select",
"date_aspi": "select", "date_aspi": "select",
"num_page": "input", "num_page": "select",
"title": "input", "title": "input",
"desc": "input", "desc": "input",
"keywords": "input", "keywords": "input",
"trackers": "select",
"rezos": "select",
"titres": "input", "titres": "input",
"liens_ext": "input",
"liens_int": "input",
"text": "input" "text": "input"
}, },
"classes":{ "classes":{
"num_page": "w80p",
"title": "w120p",
"desc": "w120p",
"keywords": "w200p",
"trackers": "w100p",
"rezos": "w100p",
"liens_int": "maxw400p",
"text": "minw400p"
} }
} #SET{id_jsonl,1} } #SET{id_jsonl,1}
<BOUCLE_corpus(DATA){source ls, #CHEMIN{tmp}/#CONFIG{corpus_web/repertoire_corpus,corpus_jsonl}/*}{par basename}> <BOUCLE_corpus(DATA){source ls, #CHEMIN{tmp}/#CONFIG{corpus_web/repertoire_corpus,corpus_jsonl}/*}{par basename}>
@ -32,17 +48,23 @@
<BOUCLE_2lignes(DATA){enum 1, #GET{nb_lignes}, #CONFIG{corpus_web/pas_corpus, 10}}{si #SIZE|<{#GET{trop_gros}}}> <BOUCLE_2lignes(DATA){enum 1, #GET{nb_lignes}, #CONFIG{corpus_web/pas_corpus, 10}}{si #SIZE|<{#GET{trop_gros}}}>
[(#SET{num_ligne,[(#COMPTEUR_BOUCLE|=={1}|?{1,#GET{nb_lignes}})]})] [(#SET{num_ligne,[(#COMPTEUR_BOUCLE|=={1}|?{1,#GET{nb_lignes}})]})]
[(#SET{content,[(#FILE|affiche_ligne{#GET{num_ligne}}|json_decode{true})]})] [(#SET{content,[(#FILE|affiche_ligne{#GET{num_ligne}}|json_decode{true})]})]
#SET{num_page,Page #VALEUR} #SET{title,''} #SET{keywords,''} #SET{description,''} #SET{titres,''} #SET{num_page,Page #VALEUR} #SET{title,''} #SET{keywords,''} #SET{description,''} #SET{titres,''} #SET{trackers,''} #SET{rezos,''} #SET{liens_ext,''} #SET{liens_int,''}
<BOUCLE_content(DATA){source table, #GET{content}}> <BOUCLE_content(DATA){source table, #GET{content}}>
[(#CLE|=={extractionContent}|oui) [(#CLE|=={extractionContent}|oui)
<BOUCLE_ss(DATA){source table, #VALEUR}> <BOUCLE_ss(DATA){source table, #VALEUR}>
[(#CLE|=={boilerpipe:text}|oui) [(#SET{text, text: [(#VALEUR|print|replace{<br>,§§}|replace{<br />,§§}|replace{<,&lt;}|replace{>,&gt;}|replace{§§,<br>}|replace{"\n", <br>})]})] ] [(#CLE|=={boilerpipe:text}|oui)
[(#CLE|=={htmlmeta:head:title}|oui) [(#SET{title, title: [(#VALEUR|print|replace{<br>,§§}|replace{<br />,§§}|replace{<,&lt;}|replace{>,&gt;}|replace{§§,<br>}|replace{"\n", <br>})]})] ] [(#SET{text, [(#VALEUR|print|supprimer_com|replace{<br>,§§}|replace{<br />,§§}|replace{<,&lt;}|replace{>,&gt;}|replace{§§,<br>}|replace{"\n", <br>})]})]
[(#CLE|=={htmlmeta:head:keywords}|oui) [(#SET{keywords, keywords: [(#VALEUR|print|replace{<br>,§§}|replace{<br />,§§}|replace{<,&lt;}|replace{>,&gt;}|replace{§§,<br>}|replace{"\n", <br>})]})] ] ]
[(#CLE|=={htmlmeta:head:description}|oui) [(#SET{description, desc: [(#VALEUR|print|replace{<br>,§§}|replace{<br />,§§}|replace{<,&lt;}|replace{>,&gt;}|replace{§§,<br>}|replace{"\n", <br>})]})] ] [(#CLE|=={htmlmeta:head:title}|oui) [(#SET{title, [(#VALEUR|print|replace{<br>,§§}|replace{<br />,§§}|replace{<,&lt;}|replace{>,&gt;}|replace{§§,<br>}|replace{"\n", <br>})]})] ]
[(#CLE|=={htmlmeta:head:keywords}|oui) [(#SET{keywords, [(#VALEUR|print|replace{<br>,§§}|replace{<br />,§§}|replace{<,&lt;}|replace{>,&gt;}|replace{§§,<br>}|replace{"\n", <br>}|replace{',', ', '})]})] ]
[(#CLE|=={htmlmeta:head:description}|oui) [(#SET{description, [(#VALEUR|print|replace{<br>,§§}|replace{<br />,§§}|replace{<,&lt;}|replace{>,&gt;}|replace{§§,<br>}|replace{"\n", <br>})]})] ]
</BOUCLE_ss> </BOUCLE_ss>
][(#CLE|=={htmlBytes}|oui) ][(#CLE|=={htmlBytes}|oui)
[(#SET{titres, [(#VALEUR|base64_decode|recup_titres)]})] [(#SET{titres, [(#VALEUR|base64_decode|recup_titres)]})]
[(#SET{trackers, [(#VALEUR|base64_decode|trouve_trackers)]})]
[(#SET{rezos, [(#VALEUR|base64_decode|trouve_rezos)]})]
[(#SET{liens_ext, [(#VALEUR|base64_decode|recup_liens_ext)]})]
[(#SET{liens_int, [(#VALEUR|base64_decode|recup_liens_int)]})]
] ]
</BOUCLE_content> </BOUCLE_content>
,{ ,{
@ -54,7 +76,11 @@
"title": [(#GET{title}|json_encode)], "title": [(#GET{title}|json_encode)],
"desc": [(#GET{description}|json_encode)], "desc": [(#GET{description}|json_encode)],
"keywords": [(#GET{keywords}|json_encode)], "keywords": [(#GET{keywords}|json_encode)],
"trackers": [(#GET{trackers}|json_encode)],
"rezos": [(#GET{rezos}|json_encode)],
"titres": [(#GET{titres}|json_encode)], "titres": [(#GET{titres}|json_encode)],
"liens_ext": [(#GET{liens_ext}|json_encode)],
"liens_int": [(#GET{liens_int}|json_encode)],
"text": [(#GET{text}|json_encode)] "text": [(#GET{text}|json_encode)]
}, },
"classes": { "classes": {
@ -68,7 +94,11 @@
"title": [(#GET{title}|json_encode)], "title": [(#GET{title}|json_encode)],
"desc": [(#GET{description}|json_encode)], "desc": [(#GET{description}|json_encode)],
"keywords": [(#GET{keywords}|json_encode)], "keywords": [(#GET{keywords}|json_encode)],
"trackers": [(#GET{trackers}|json_encode)],
"rezos": [(#GET{rezos}|json_encode)],
"titres": [(#GET{titres}|json_encode)], "titres": [(#GET{titres}|json_encode)],
"liens_ext": [(#GET{liens_ext}|json_encode)],
"liens_int": [(#GET{liens_int}|json_encode)],
"text": [(#GET{text}|json_encode)] "text": [(#GET{text}|json_encode)]
} }
} [(#SET{id_jsonl, #GET{id_jsonl}|plus{1}})] } [(#SET{id_jsonl, #GET{id_jsonl}|plus{1}})]