path: root/Makefile

PATH := $(PATH):/usr/local/bin

.PHONY: all
all: enwiktionary-latest.version
all: enwiktionary-latest.index
all: enwiki-latest.version
all: enwiki-latest.index
all: phraser-words500K.index
all: phraser-phrases5M.index

.PHONY: newest
newest:
	rm -f *.version && make

.PHONY: reindex
reindex:
	rm -f *.index && make

.PHONY: sync
sync: newest
	rsync -av *.index tilde:public_html/pzzl.org/nut/

.PHONY: trim
trim:
	ls enwiki-*-snap.index | sort | head -n -2 | xargs -rt rm
	ls enwiktionary-*-snap.index | sort | head -n -2 | xargs -rt rm

%/.git:
	git submodule update --init $*

nutrimatic/bin: nutrimatic/.git
	cd nutrimatic && ./build.py

wikiextractor/WikiExtractor.py: wikiextractor/.git

%-latest.version:
	wget -O - https://dumps.wikimedia.org/$*/latest/$*-latest-pages-articles.xml.bz2-rss.xml |\
		xpath '//rss/channel/link/text()' |\
		sed 's/^.*\///' > $@

enwiki-%-snap.merge: nutrimatic/bin wikiextractor/WikiExtractor.py
	# ~7h30m
	wget -O - https://dumps.wikimedia.org/enwiki/$*/enwiki-$*-pages-articles.xml.bz2 |\
		bzip2 -d |\
		python wikiextractor/WikiExtractor.py -q -o- - |\
		./merge.sh $@

enwiktionary-%-snap.merge: nutrimatic/bin wikiextractor/WikiExtractor.py
	# ~1h
	wget -O - https://dumps.wikimedia.org/enwiktionary/$*/enwiktionary-$*-pages-articles.xml.bz2 |\
		bzip2 -d |\
		python wikiextractor/WikiExtractor.py -q -o- - |\
		./merge.sh $@

%-snap.index: %-snap.merge nutrimatic/bin
	nutrimatic/bin/merge-indexes 5 $*-snap.merge $@

%-latest.index: %-latest.version
	make $*-$(shell cat $*-latest.version)-snap.index
	ln -sf $*-$(shell cat $*-latest.version)-snap.index $@

phraser-words500K.index: nutrimatic/bin
	# ~ 3m
	wget -O - https://lahosken.san-francisco.ca.us/frivolity/prog/phraser/words_500K.txt |\
		awk -F'\t' '{ for (i=0; i < $$1; i++) print $$2 }' |\
		./merge.sh $@

phraser-phrases5M.index: nutrimatic/bin
	# ~ 10m
	wget -O - https://lahosken.san-francisco.ca.us/frivolity/prog/phraser/phrases_5M.txt |\
		awk -F'\t' '{ for (i=0; i < $$1; i++) print $$2 }' |\
		./merge.sh $@