republique

commit cfc5ded7132074be9341112e5b10addf3f7cf17c
parent 26651fb2c35c43919d36c0651fa4a02d24db3122
Author: Antoine Amarilli <a3nm@a3nm.net>
Date:   Wed, 14 Oct 2015 12:21:33 +0200

dump

Diffstat:
.gitignore
 | 
1
+
README
 | 
4
++++
dump.sh
 | 
34
++++++++++++++++++++++++++++++++++
dump_all.sh
 | 
17
+++++++++++++++++

4 files changed, 56 insertions(+), 0 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -1 +1,2 @@
 __pycache__/
+dump/
diff --git a/README b/README
@@ -18,6 +18,10 @@ Requires Debian packages python3-requests, python3-bs4, python3-lxml
   connects to www.republique-numerique.fr local account EMAIL PASSWORD
   and performs votes given on standard input in the form of get_votes.py
 
+- dump_all.sh DEST
+  dump raw JSON of all opinions and versions to DEST/opinions and DEST/versions
+  (also dumps a few ones that don't really exist and must be filtered)
+
 Examples:
 
 - to perform the same votes as USER
diff --git a/dump.sh b/dump.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# dump URL $1/$i to $2/$i starting at $i=$3 until $4 consecutive without $5
+# no trailing slashes, uses JSON
+
+URL="$1"
+DEST="$2"
+NUM=$(($3 - 1))
+LIMIT="$4" # max consecutive failures
+EXPR="$5"
+
+FAIL="0"
+
+mkdir -p "$DEST"
+
+while true
+do
+  NUM=$(($NUM + 1))
+  curl -s -H "Accept: application/json" "$URL/$NUM" > "$DEST/$NUM"
+  echo "retrieved $NUM"
+  sleep 1
+  if grep "$EXPR" "$DEST/$NUM" > /dev/null
+  then
+    FAIL="0"
+  else
+    FAIL=$(($FAIL + 1))
+  fi
+  if [ "$FAIL" -gt "$LIMIT" ]
+  then
+    break
+  fi
+done
+
+
diff --git a/dump_all.sh b/dump_all.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+# dump all opinions and versions to $1/
+
+DIR="$( cd "$( dirname "$0" )" && pwd )"
+cd "$DIR"
+
+DEST="$1"
+mkdir -p "$DEST/opinions" "$DEST/versions"
+
+./dump.sh "https://www.republique-numerique.fr/api/opinions" \
+  "$DEST/opinions" 1 100 title
+
+# actually, /opinions/$i/versions seems to contain all versions
+# no matter the opinion
+./dump.sh "https://www.republique-numerique.fr/api/opinions/61/versions" \
+  "$DEST/versions" 1 100 title 
+