Enhancement: More information about S3 keys searched for and Dockerfi…

…le that uses submodule instead of channel (#1151)

* Eric's Dockerfile improvements
* Update misc/e2elive.py

misc/Dockerfile

@@ -8,21 +8,17 @@ ENV HOME /opt
 ENV DEBIAN_FRONTEND noninteractive
 RUN apt-get update && apt-get install -y apt-utils curl git git-core bsdmainutils python3 python3-pip make bash libtool libboost-math-dev libffi-dev
 
-# Install algod nightly binaries to the path
-RUN mkdir -p /opt/algorand/{bin,data}
-ADD https://github.com/algorand/go-algorand/raw/1e1474216421da27008726c44ebe0a5ba2fb6a08/cmd/updater/update.sh /opt/algorand/bin/update.sh
-RUN chmod 755 /opt/algorand/bin/update.sh
-WORKDIR /opt/algorand/bin
-RUN ./update.sh -i -c nightly -p /opt/algorand/bin -d /opt/algorand/data -n
-RUN find /opt/algorand
-ENV PATH="/opt/algorand/bin:${PATH}"
-
 # Setup files for test
 RUN mkdir -p /opt/go/indexer
 COPY . /opt/go/indexer
 WORKDIR /opt/go/indexer
+RUN rm -f $HOME/go/bin/algod
 RUN rm /opt/go/indexer/cmd/algorand-indexer/algorand-indexer
+WORKDIR /opt/go/indexer/third_party/go-algorand
+RUN make install
+WORKDIR /opt/go/indexer
 RUN make
+ENV PATH="${HOME}/go/bin/:${PATH}"
 RUN pip3 install -r misc/requirements.txt
 
 ENV INDEXER_DATA="${HOME}/indexer/"

misc/e2elive.py

@@ -77,7 +77,12 @@ def main():
         s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
         tarname = "net_done.tar.bz2"
         tarpath = os.path.join(tempdir, tarname)
-        firstFromS3Prefix(s3, bucket, "indexer/e2e4", tarname, outpath=tarpath)
+        prefix = "indexer/e2e4"
+        success = firstFromS3Prefix(s3, bucket, prefix, tarname, outpath=tarpath)
+        if not success:
+            raise Exception(
+                f"failed to locate tarname={tarname} from AWS S3 path {bucket}/{prefix}"
+            )
         source_is_tar = True
         sourcenet = tarpath
     tempnet = os.path.join(tempdir, "net")

misc/util.py

@@ -157,12 +157,30 @@ def ensure_test_db(connection_string, keep_temps=False):
 
 
 # whoever calls this will need to import boto and get the s3 client
-def firstFromS3Prefix(s3, bucket, prefix, desired_filename, outdir=None, outpath=None):
+def firstFromS3Prefix(
+    s3, bucket, prefix, desired_filename, outdir=None, outpath=None
+) -> bool:
+    haystack = []
+    found_needle = False
+
+    def report():
+        print(
+            f"""Hello from firstFromS3Prefix() !!!!
+        I finished searching for the needle {desired_filename} in the AWS S3 path {bucket}/{prefix}.
+        haystack={haystack}
+        len(haystack)={len(haystack)}
+        found_needle={found_needle}
+"""
+        )
+
+    atexit.register(report)
+
     response = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, MaxKeys=50)
     if (not response.get("KeyCount")) or ("Contents" not in response):
         raise Exception("nothing found in s3://{}/{}".format(bucket, prefix))
     for x in response["Contents"]:
         path = x["Key"]
+        haystack.append(path)
         _, fname = path.rsplit("/", 1)
         if fname == desired_filename:
             if outpath is None:
@@ -171,5 +189,8 @@ def firstFromS3Prefix(s3, bucket, prefix, desired_filename, outdir=None, outpath
                 outpath = os.path.join(outdir, desired_filename)
             logger.info("s3://%s/%s -> %s", bucket, x["Key"], outpath)
             s3.download_file(bucket, x["Key"], outpath)
-            return
+            found_needle = True
+            break
+
     logger.warning("file not found in s3://{}/{}".format(bucket, prefix))
+    return found_needle