Feat: ajout de l'extraction des liens internes et externes
This commit is contained in:
cy_altern
parent
20663f23ff
commit
504e7affa4
3 changed files with 210 additions and 8 deletions
|
|
@ -45,6 +45,23 @@
|
|||
<div class="aff_masq_">Titres :</div>
|
||||
<div class="details_">[(#VALEUR|base64_decode|recup_titres)]</div>
|
||||
</li>
|
||||
<li>
|
||||
<div class="aff_masq_">Trackers :</div>
|
||||
<div class="details_">[(#VALEUR|base64_decode|trouve_trackers)]</div>
|
||||
</li>
|
||||
<li>
|
||||
<div class="aff_masq_">Rezos socios :</div>
|
||||
<div class="details_">[(#VALEUR|base64_decode|trouve_rezos)]</div>
|
||||
</li>
|
||||
<li>
|
||||
<div class="aff_masq_">Liens externes :</div>
|
||||
<div class="details_">[(#VALEUR|base64_decode|recup_liens_ext)]</div>
|
||||
</li>
|
||||
<li>
|
||||
<div class="aff_masq_">Liens internes :</div>
|
||||
<div class="details_">[(#VALEUR|base64_decode|recup_liens_int)]</div>
|
||||
</li>
|
||||
|
||||
]
|
||||
[(#CLE|=={extractionContent}|oui)
|
||||
<BOUCLE_ss(DATA){source table, #VALEUR}>
|
||||
|
|
@ -52,7 +69,7 @@
|
|||
<li>
|
||||
<div class="aff_masq_">[(#CLE|replace{htmlmeta:head:, meta }|replace{boilerpipe:, ''}|ucfirst)] : </div>
|
||||
<div class="details_">
|
||||
[(#VALEUR|print|replace{<br>,§§}|replace{<br />,§§}|replace{<,<}|replace{>,>}|replace{§§,<br>}|replace{"\n", <br>})]
|
||||
[(#VALEUR|print|supprimer_com|replace{<br>,§§}|replace{<br />,§§}|replace{<,<}|replace{>,>}|replace{§§,<br>}|replace{"\n", <br>})]
|
||||
</div>
|
||||
</li>
|
||||
]
|
||||
|
|
|
|||
|
|
@ -192,3 +192,158 @@ function recup_aspi($nom_fichier_jsonl) {
|
|||
$T = explode('-', $T);
|
||||
return $T[0].'-'.$T[1].'-'.$T[2];
|
||||
}
|
||||
|
||||
function supprimer_com($boiler_text) {
|
||||
return preg_replace('/Poster un nouveau commentaire.*/si', '', $boiler_text);
|
||||
}
|
||||
|
||||
function trouve_trackers($html) {
|
||||
// google(-?)analytics|xiti.com|ranktrackr.net
|
||||
$trackers = [];
|
||||
preg_match_all('/google(-?)analytics|xiti\.com|ranktrackr\.net|seoposition\.com/si', $html, $match, PREG_SET_ORDER);
|
||||
if ($match) {
|
||||
foreach($match as $m) {
|
||||
if (strtolower(str_replace(['-', ' '], '', $m[0])) === 'googleanalytics') {
|
||||
if (!in_array('google analytics', $trackers)) {
|
||||
$trackers[] = 'google analytics';
|
||||
}
|
||||
} else {
|
||||
if (!in_array(strtolower($m[0]), $trackers)) {
|
||||
$trackers[] = strtolower($m[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return count($trackers) ? "<ul><li>".join("</li><li>", $trackers)."</li></ul>" : '';
|
||||
}
|
||||
|
||||
function trouve_rezos($html) {
|
||||
// facebook.com|twitter.com|pinterest.com|plusone
|
||||
$rezos = [];
|
||||
preg_match_all('/facebook\.com|twitter\.com|plusone/si', $html, $match, PREG_SET_ORDER);
|
||||
if ($match) {
|
||||
foreach($match as $m) {
|
||||
switch (strtolower($m[0])) {
|
||||
case 'facebook.com':
|
||||
if (!in_array('Facebook', $rezos)) {
|
||||
$rezos[] = 'Facebook';
|
||||
}
|
||||
break;
|
||||
case 'twitter.com':
|
||||
if (!in_array('Twitter', $rezos)) {
|
||||
$rezos[] = 'Twitter';
|
||||
}
|
||||
break;
|
||||
case 'plusone':
|
||||
if (!in_array('Google Plus One', $rezos)) {
|
||||
$rezos[] = 'Google Plus One';
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return count($rezos) ? "<ul><li>".join("</li><li>", $rezos)."</li></ul>" : '';
|
||||
}
|
||||
|
||||
function recup_liens_ext($html) {
|
||||
// <a href="http(s?)://(?!www.o2zone.tv|t.co)([^"]*)
|
||||
$liens_ext = [];
|
||||
$exclus = [
|
||||
'w3.org',
|
||||
'adobe.com',
|
||||
'xiti.com',
|
||||
'ranktrackr.net',
|
||||
'seoposition.com',
|
||||
'facebook.com',
|
||||
'twitter.com',
|
||||
];
|
||||
preg_match_all('/<a href="http(s?):\/\/(?!www\.o2zone.tv|t\.co)([^"]*)/si', $html, $match, PREG_SET_ORDER);
|
||||
if ($match) {
|
||||
foreach($match as $m) {
|
||||
// domaines exclus
|
||||
$domaine = explode('/', $m[2])[0];
|
||||
if (substr_count($domaine, '.') > 1) {
|
||||
$Tdomaine = explode('.', $domaine);
|
||||
$domaine = array_pop($Tdomaine);
|
||||
$domaine = array_pop($Tdomaine).'.'.$domaine;
|
||||
}
|
||||
if(in_array($domaine, $exclus)) {
|
||||
continue;
|
||||
}
|
||||
if (!in_array($m[2], $liens_ext)) {
|
||||
$liens_ext[] = $m[2];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return count($liens_ext) ? "<ul><li>".join("</li><li>", $liens_ext)."</li></ul>" : '';
|
||||
}
|
||||
|
||||
function recup_liens_int($html) {
|
||||
// <a href="http(s?)://(?!www.o2zone.tv|t.co)([^"]*)
|
||||
$liens_int = [];
|
||||
$txt_exclus = [
|
||||
'Accueil',
|
||||
'Edito',
|
||||
'En savoir plus...',
|
||||
'Plus de mots-clés',
|
||||
'Lire la suite »',
|
||||
' Lire la suite »',
|
||||
' Lire la suite »',
|
||||
'Ajouter un commentaire',
|
||||
'dernier »',
|
||||
'suivant ›',
|
||||
'XHTML',
|
||||
'répondre',
|
||||
'Téléchargez le plus récent lecteur Flash',
|
||||
'1',
|
||||
'2',
|
||||
'3',
|
||||
'4',
|
||||
'5',
|
||||
'6',
|
||||
'7',
|
||||
'8',
|
||||
'9',
|
||||
'Partie1',
|
||||
'Partie2',
|
||||
'Partie3',
|
||||
'Partie4',
|
||||
];
|
||||
$url_exclus = [
|
||||
'/',
|
||||
'/la-tele',
|
||||
'/video',
|
||||
'/la-boite-outils',
|
||||
'/lassociation',
|
||||
'/forum',
|
||||
'/user',
|
||||
'/aide-du-site-o2zone',
|
||||
'/propos',
|
||||
'/filter/tips',
|
||||
'/rss.xml',
|
||||
];
|
||||
preg_match_all('/<a href="(http(s?):\/\/www\.o2zone\.tv)?(\/[^"]*)?([^>]*)?">(<(img|p|acronym)[^>]*>)?(.*?)<\/a>/si', $html, $match, PREG_SET_ORDER);
|
||||
if ($match) {
|
||||
foreach($match as $m) {
|
||||
$m[7] = preg_replace('/<(img|(\/)?span)[^>]*>/si', '', $m[7]);
|
||||
// liens exclus
|
||||
if(in_array($m[3], $url_exclus) || str_contains($m[3], '#comment') || str_contains($m[3], '/user/') || in_array($m[7], $txt_exclus) || str_replace([' ', ' '], '', $m[7]) === '') {
|
||||
continue;
|
||||
}
|
||||
if (!array_key_exists($m[3], $liens_int)) {
|
||||
$liens_int[$m[3]] = $m[7];
|
||||
}
|
||||
}
|
||||
}
|
||||
$l_int = [];
|
||||
if (count($liens_int)) {
|
||||
foreach ($liens_int as $url => $titre) {
|
||||
$l_int[] = $url.' : '.$titre;
|
||||
}
|
||||
}
|
||||
|
||||
return count($l_int) ? "<ul><li>".join("</li><li>", $l_int)."</li></ul>" : '';
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,20 +9,36 @@
|
|||
"title": "Meta title",
|
||||
"desc": "Meta description",
|
||||
"keywords": "Meta keywords",
|
||||
"trackers": "Trackers",
|
||||
"rezos": "Rézos socios",
|
||||
"titres": "titres",
|
||||
"liens_ext": "Liens externes",
|
||||
"liens_int": "Liens internes",
|
||||
"text": "Texte"
|
||||
},
|
||||
"filtreCol" : {
|
||||
"site" : "select",
|
||||
"date_aspi": "select",
|
||||
"num_page": "input",
|
||||
"num_page": "select",
|
||||
"title": "input",
|
||||
"desc": "input",
|
||||
"keywords": "input",
|
||||
"trackers": "select",
|
||||
"rezos": "select",
|
||||
"titres": "input",
|
||||
"liens_ext": "input",
|
||||
"liens_int": "input",
|
||||
"text": "input"
|
||||
},
|
||||
"classes":{
|
||||
"num_page": "w80p",
|
||||
"title": "w120p",
|
||||
"desc": "w120p",
|
||||
"keywords": "w200p",
|
||||
"trackers": "w100p",
|
||||
"rezos": "w100p",
|
||||
"liens_int": "maxw400p",
|
||||
"text": "minw400p"
|
||||
}
|
||||
} #SET{id_jsonl,1}
|
||||
<BOUCLE_corpus(DATA){source ls, #CHEMIN{tmp}/#CONFIG{corpus_web/repertoire_corpus,corpus_jsonl}/*}{par basename}>
|
||||
|
|
@ -32,17 +48,23 @@
|
|||
<BOUCLE_2lignes(DATA){enum 1, #GET{nb_lignes}, #CONFIG{corpus_web/pas_corpus, 10}}{si #SIZE|<{#GET{trop_gros}}}>
|
||||
[(#SET{num_ligne,[(#COMPTEUR_BOUCLE|=={1}|?{1,#GET{nb_lignes}})]})]
|
||||
[(#SET{content,[(#FILE|affiche_ligne{#GET{num_ligne}}|json_decode{true})]})]
|
||||
#SET{num_page,Page #VALEUR} #SET{title,''} #SET{keywords,''} #SET{description,''} #SET{titres,''}
|
||||
#SET{num_page,Page #VALEUR} #SET{title,''} #SET{keywords,''} #SET{description,''} #SET{titres,''} #SET{trackers,''} #SET{rezos,''} #SET{liens_ext,''} #SET{liens_int,''}
|
||||
<BOUCLE_content(DATA){source table, #GET{content}}>
|
||||
[(#CLE|=={extractionContent}|oui)
|
||||
<BOUCLE_ss(DATA){source table, #VALEUR}>
|
||||
[(#CLE|=={boilerpipe:text}|oui) [(#SET{text, text: [(#VALEUR|print|replace{<br>,§§}|replace{<br />,§§}|replace{<,<}|replace{>,>}|replace{§§,<br>}|replace{"\n", <br>})]})] ]
|
||||
[(#CLE|=={htmlmeta:head:title}|oui) [(#SET{title, title: [(#VALEUR|print|replace{<br>,§§}|replace{<br />,§§}|replace{<,<}|replace{>,>}|replace{§§,<br>}|replace{"\n", <br>})]})] ]
|
||||
[(#CLE|=={htmlmeta:head:keywords}|oui) [(#SET{keywords, keywords: [(#VALEUR|print|replace{<br>,§§}|replace{<br />,§§}|replace{<,<}|replace{>,>}|replace{§§,<br>}|replace{"\n", <br>})]})] ]
|
||||
[(#CLE|=={htmlmeta:head:description}|oui) [(#SET{description, desc: [(#VALEUR|print|replace{<br>,§§}|replace{<br />,§§}|replace{<,<}|replace{>,>}|replace{§§,<br>}|replace{"\n", <br>})]})] ]
|
||||
[(#CLE|=={boilerpipe:text}|oui)
|
||||
[(#SET{text, [(#VALEUR|print|supprimer_com|replace{<br>,§§}|replace{<br />,§§}|replace{<,<}|replace{>,>}|replace{§§,<br>}|replace{"\n", <br>})]})]
|
||||
]
|
||||
[(#CLE|=={htmlmeta:head:title}|oui) [(#SET{title, [(#VALEUR|print|replace{<br>,§§}|replace{<br />,§§}|replace{<,<}|replace{>,>}|replace{§§,<br>}|replace{"\n", <br>})]})] ]
|
||||
[(#CLE|=={htmlmeta:head:keywords}|oui) [(#SET{keywords, [(#VALEUR|print|replace{<br>,§§}|replace{<br />,§§}|replace{<,<}|replace{>,>}|replace{§§,<br>}|replace{"\n", <br>}|replace{',', ', '})]})] ]
|
||||
[(#CLE|=={htmlmeta:head:description}|oui) [(#SET{description, [(#VALEUR|print|replace{<br>,§§}|replace{<br />,§§}|replace{<,<}|replace{>,>}|replace{§§,<br>}|replace{"\n", <br>})]})] ]
|
||||
</BOUCLE_ss>
|
||||
][(#CLE|=={htmlBytes}|oui)
|
||||
[(#SET{titres, [(#VALEUR|base64_decode|recup_titres)]})]
|
||||
[(#SET{trackers, [(#VALEUR|base64_decode|trouve_trackers)]})]
|
||||
[(#SET{rezos, [(#VALEUR|base64_decode|trouve_rezos)]})]
|
||||
[(#SET{liens_ext, [(#VALEUR|base64_decode|recup_liens_ext)]})]
|
||||
[(#SET{liens_int, [(#VALEUR|base64_decode|recup_liens_int)]})]
|
||||
]
|
||||
</BOUCLE_content>
|
||||
,{
|
||||
|
|
@ -54,7 +76,11 @@
|
|||
"title": [(#GET{title}|json_encode)],
|
||||
"desc": [(#GET{description}|json_encode)],
|
||||
"keywords": [(#GET{keywords}|json_encode)],
|
||||
"trackers": [(#GET{trackers}|json_encode)],
|
||||
"rezos": [(#GET{rezos}|json_encode)],
|
||||
"titres": [(#GET{titres}|json_encode)],
|
||||
"liens_ext": [(#GET{liens_ext}|json_encode)],
|
||||
"liens_int": [(#GET{liens_int}|json_encode)],
|
||||
"text": [(#GET{text}|json_encode)]
|
||||
},
|
||||
"classes": {
|
||||
|
|
@ -68,7 +94,11 @@
|
|||
"title": [(#GET{title}|json_encode)],
|
||||
"desc": [(#GET{description}|json_encode)],
|
||||
"keywords": [(#GET{keywords}|json_encode)],
|
||||
"trackers": [(#GET{trackers}|json_encode)],
|
||||
"rezos": [(#GET{rezos}|json_encode)],
|
||||
"titres": [(#GET{titres}|json_encode)],
|
||||
"liens_ext": [(#GET{liens_ext}|json_encode)],
|
||||
"liens_int": [(#GET{liens_int}|json_encode)],
|
||||
"text": [(#GET{text}|json_encode)]
|
||||
}
|
||||
} [(#SET{id_jsonl, #GET{id_jsonl}|plus{1}})]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue