republique

helper scripts for www.republique-numerique.fr
git clone https://a3nm.net/git/republique/
Log | Files | Refs | README

commit cfc5ded7132074be9341112e5b10addf3f7cf17c
parent 26651fb2c35c43919d36c0651fa4a02d24db3122
Author: Antoine Amarilli <a3nm@a3nm.net>
Date:   Wed, 14 Oct 2015 12:21:33 +0200

dump

Diffstat:
.gitignore | 1+
README | 4++++
dump.sh | 34++++++++++++++++++++++++++++++++++
dump_all.sh | 17+++++++++++++++++
4 files changed, 56 insertions(+), 0 deletions(-)

diff --git a/.gitignore b/.gitignore @@ -1 +1,2 @@ __pycache__/ +dump/ diff --git a/README b/README @@ -18,6 +18,10 @@ Requires Debian packages python3-requests, python3-bs4, python3-lxml connects to www.republique-numerique.fr local account EMAIL PASSWORD and performs votes given on standard input in the form of get_votes.py +- dump_all.sh DEST + dump raw JSON of all opinions and versions to DEST/opinions and DEST/versions + (also dumps a few ones that don't really exist and must be filtered) + Examples: - to perform the same votes as USER diff --git a/dump.sh b/dump.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# dump URL $1/$i to $2/$i starting at $i=$3 until $4 consecutive without $5 +# no trailing slashes, uses JSON + +URL="$1" +DEST="$2" +NUM=$(($3 - 1)) +LIMIT="$4" # max consecutive failures +EXPR="$5" + +FAIL="0" + +mkdir -p "$DEST" + +while true +do + NUM=$(($NUM + 1)) + curl -s -H "Accept: application/json" "$URL/$NUM" > "$DEST/$NUM" + echo "retrieved $NUM" + sleep 1 + if grep "$EXPR" "$DEST/$NUM" > /dev/null + then + FAIL="0" + else + FAIL=$(($FAIL + 1)) + fi + if [ "$FAIL" -gt "$LIMIT" ] + then + break + fi +done + + diff --git a/dump_all.sh b/dump_all.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# dump all opinions and versions to $1/ + +DIR="$( cd "$( dirname "$0" )" && pwd )" +cd "$DIR" + +DEST="$1" +mkdir -p "$DEST/opinions" "$DEST/versions" + +./dump.sh "https://www.republique-numerique.fr/api/opinions" \ + "$DEST/opinions" 1 100 title + +# actually, /opinions/$i/versions seems to contain all versions +# no matter the opinion +./dump.sh "https://www.republique-numerique.fr/api/opinions/61/versions" \ + "$DEST/versions" 1 100 title +