republique

commit 939bf57395e0c5342c20c3d435ebd9226b550320
parent f496937a97c0be04c185996c9c3588116326c7e1
Author: Antoine Amarilli <a3nm@a3nm.net>
Date:   Mon, 19 Oct 2015 22:29:41 +0200

retry fetching when it unexplainably fails

Diffstat:
dump.sh
 | 
43
+++++++++++++++++++++++++++++++++----------

1 file changed, 33 insertions(+), 10 deletions(-)
diff --git a/dump.sh b/dump.sh
@@ -8,6 +8,7 @@ DEST="$2"
 NUM=$(($3 - 1))
 LIMIT="$4" # max consecutive failures
 EXPR="$5"
+MAX_RETRIES="5"
 
 FAIL="0"
 
@@ -16,18 +17,40 @@ mkdir -p "$DEST"
 while true
 do
   NUM=$(($NUM + 1))
-  curl -s -H "Accept: application/json" "$URL/$NUM" > "$DEST/$NUM"
-  echo "retrieved $NUM"
-  sleep 1
-  if grep "$EXPR" "$DEST/$NUM" > /dev/null
+  RETRIES="0"
+  while true
+  do
+    curl -s -H "Accept: application/json" "$URL/$NUM" > "$DEST/$NUM"
+    sleep 1
+    if [ -s "$DEST/$NUM" ]
+    then
+      break
+    fi
+    if [ $RETRIES -gt $MAX_RETRIES ]
+    then
+      # too many attempts, give up
+      rm -f "$DEST/$NUM"
+      break
+    fi
+    # transient fetch problems happen
+    RETRIES=$(($RETRIES + 1))
+    echo "error fetching $DEST/$NUM, retrying (${RETRIES}/${MAX_RETRIES})"
+  done
+  if [ ! -f "$DEST/$NUM" ]
   then
-    FAIL="0"
+    echo "could not fetch $DEST/$NUM, removed it"
   else
-    FAIL=$(($FAIL + 1))
-  fi
-  if [ "$FAIL" -gt "$LIMIT" ]
-  then
-    break
+    echo "retrieved $DEST/$NUM"
+    if grep "$EXPR" "$DEST/$NUM" > /dev/null
+    then
+      FAIL="0"
+    else
+      FAIL=$(($FAIL + 1))
+    fi
+    if [ "$FAIL" -gt "$LIMIT" ]
+    then
+      break
+    fi
   fi
 done