republique

helper scripts for www.republique-numerique.fr
git clone https://a3nm.net/git/republique/
Log | Files | Refs | README

commit 0f7614e0709f740803d3657d3a8793bdd8e593aa
parent cc159c20792ddd923f0aa0996e56d8d8994f8ba2
Author: Antoine Amarilli <a3nm@a3nm.net>
Date:   Sun, 11 Oct 2015 01:35:19 +0200

continue

Diffstat:
common.py | 28++++++++++++++++++++++++++++
get_propositions.py | 37+++++++++++++++++++++++++++++++++++++
get_votes.py | 46++++++++++++++++++++++++++++++++++++++++++++++
vote.py | 2+-
4 files changed, 112 insertions(+), 1 deletion(-)

diff --git a/common.py b/common.py @@ -0,0 +1,28 @@ +#!/bin/python3 + +from bs4 import BeautifulSoup +import requests + +HEADERS = { 'User-Agent': 'Mozilla' } +URL = 'https://www.republique-numerique.fr%s' + +def url2res(relurl): + """get identifier of URL""" + # this sucks but I don't know how else to do it + url = URL % relurl + data = requests.get(url, headers=HEADERS) + tree = BeautifulSoup(data.text, 'html.parser') + divs = (tree.find_all('div', id='render-opinion') + + tree.find_all('div', id='render-opinion-version')) + div = divs[0] + opinion = div.get('data-opinion') + version = None + try: + version = div.get('data-version') + except KeyError: + pass + if version: + return '/opinions/%s/versions/%s' % (opinion, version) + else: + return '/opinions/%s' % opinion + diff --git a/get_propositions.py b/get_propositions.py @@ -0,0 +1,37 @@ +#!/usr/bin/python3 +# Get opinions and modifications of a user + +from common import HEADERS, url2res +from bs4 import BeautifulSoup +import requests +import sys + +PROPOSITIONS_URL = 'https://www.republique-numerique.fr/profile/%s/opinions' +VERSIONS_URL = 'https://www.republique-numerique.fr/profile/%s/versions' + +if __name__ == '__main__': + try: + user = sys.argv[1] + except IndexError: + print("Usage: %s USER\n" + "Returns all opinions and modifications of USER" % + sys.argv[0], file=sys.stderr) + sys.exit(1) + + for url in [PROPOSITIONS_URL % user, VERSIONS_URL % user]: + data = requests.get(url, headers=HEADERS) + tree = BeautifulSoup(data.text, 'html.parser') + + for div in tree.find_all('div', class_='opinion__data'): + res_url = None + for a in div.find_all('a'): + v = a.get('href') + if not v: + continue + if v.startswith('/consultations'): + res_url = a.get('href') + break + + print ("%s" % url2res(res_url)) + + diff --git a/get_votes.py b/get_votes.py @@ -0,0 +1,46 @@ +#!/usr/bin/python3 +# Get all votes of a user + +from common import HEADERS, url2res +from bs4 import BeautifulSoup +import requests +import sys + +VOTE_URL = 'https://www.republique-numerique.fr/profile/%s/votes' + +KEYS = { + 'success': 1, + 'warning': 0, + 'danger': -1, + } + +if __name__ == '__main__': + try: + user = sys.argv[1] + except IndexError: + print("Usage: %s USER\n" + "Returns all votes of USER" % + sys.argv[0], file=sys.stderr) + sys.exit(1) + + data = requests.get(VOTE_URL % user, headers=HEADERS) + votes_tree = BeautifulSoup(data.text, 'html.parser') + + for div in votes_tree.find_all('div', class_='opinion__data'): + res_url = None + for a in div.find_all('a'): + v = a.get('href') + if not v: + continue + if v.startswith('/consultations'): + res_url = a.get('href') + break + raw_v = None + for span in div.find_all('span'): + raw_v = span.get('class') + break + v = KEYS[raw_v[1].split('-')[1]] + + print ("%s %s" % (url2res(res_url), v)) + + diff --git a/vote.py b/vote.py @@ -6,8 +6,8 @@ from bs4 import BeautifulSoup import json import requests import sys +from common import HEADERS -HEADERS = { 'User-Agent': 'Mozilla' } HEADERS_JSON = { 'Accept': "application/json" }