From 1f490b6ff954a433cecaa17bc74c9f2543edcebb Mon Sep 17 00:00:00 2001 From: Kevin Wallace Date: Sat, 25 Jan 2025 18:28:52 -0800 Subject: use wikiextractor from local env, fetch intermediate .bz2, `xpath -e` --- Makefile | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 398a518..95d78f5 100644 --- a/Makefile +++ b/Makefile @@ -22,8 +22,10 @@ sync: newest .PHONY: trim trim: - ls enwiki-*-snap.index | sort | head -n -2 | xargs -rt rm - ls enwiktionary-*-snap.index | sort | head -n -2 | xargs -rt rm + ls enwiki-*-snap.index | sort | head -n -2 | xargs -t rm + ls enwiktionary-*-snap.index | sort | head -n -2 | xargs -t rm + ls enwiki-*-pages-articles.xml.bz2 | sort | head -n -2 | xargs -t rm + ls enwiktionary-*-pages-articles.xml.bz2 | sort | head -n -2 | xargs -t rm %/.git: git submodule update --init $* @@ -31,25 +33,24 @@ trim: nutrimatic/bin: nutrimatic/.git cd nutrimatic && ./build.py -wikiextractor/WikiExtractor.py: wikiextractor/.git - %-latest.version: wget -O - https://dumps.wikimedia.org/$*/latest/$*-latest-pages-articles.xml.bz2-rss.xml |\ - xpath '//rss/channel/link/text()' |\ + xpath -e '//rss/channel/link/text()' |\ sed 's/^.*\///' > $@ -enwiki-%-snap.merge: nutrimatic/bin wikiextractor/WikiExtractor.py - # ~7h30m - wget -O - https://dumps.wikimedia.org/enwiki/$*/enwiki-$*-pages-articles.xml.bz2 |\ - bzip2 -d |\ - python wikiextractor/WikiExtractor.py -q -o- - |\ - ./merge.sh $@ +enwiki-%-pages-articles.xml.bz2: + wget -O $@ https://dumps.wikimedia.org/enwiki/$*/enwiki-$*-pages-articles.xml.bz2 -enwiktionary-%-snap.merge: nutrimatic/bin wikiextractor/WikiExtractor.py +enwiki-%-snap.merge: nutrimatic/bin enwiki-%-pages-articles.xml.bz2 # ~1h - wget -O - https://dumps.wikimedia.org/enwiktionary/$*/enwiktionary-$*-pages-articles.xml.bz2 |\ - bzip2 -d |\ - python wikiextractor/WikiExtractor.py -q -o- - |\ + wikiextractor -q -o- enwiki-$*-pages-articles.xml.bz2 |\ + ./merge.sh $@ + +enwiktionary-%-pages-articles.xml.bz2: + wget -O $@ https://dumps.wikimedia.org/enwiktionary/$*/enwiktionary-$*-pages-articles.xml.bz2 + +enwiktionary-%-snap.merge: nutrimatic/bin enwiktionary-%-pages-articles.xml.bz2 + wikiextractor -q -o- enwiktionary-$*-pages-articles.xml.bz2 |\ ./merge.sh $@ %-snap.index: %-snap.merge nutrimatic/bin -- cgit v1.2.3