@@ 1,21 1,29 @@
#!/usr/bin/env python
-import subprocess, os, platform, io, httplib, urllib, sys, json, traceback, datetime, time, base64, glob, re, shutil
+import subprocess, os, platform, io, httplib, urllib, sys, json, traceback, datetime, time, base64, glob, re, shutil, itertools
+import config
-import config
+print "Starting CouchBot"
if len(sys.argv) < 2:
bot = config.init_bucketbot("config.json", [])
else:
bot = config.init_bucketbot(sys.argv[1], sys.argv[2:])
+print "Server : %s://%s/%s\nAuth : %s" % (bot.db_scheme, bot.db_server, bot.db_name, bot.user + ":" + "*" * len(bot.passwd))
+print "Local : %s\nRemote : %s" % (bot.local, bot.remote)
+
# make work directories for source and building
if not os.path.isdir(bot.workspace):
os.makedirs(bot.workspace)
+print "Workspace: %s" % bot.workspace
+print "PR Tests : %s" % bot.test_branch
+print "Releases : %s" % bot.release_branch
+
if not os.path.isdir(bot.local):
os.makedirs(bot.local)
- subprocess.check_call(["hg", "clone", bot.remote, bot.local], stdout = subprocess.PIPE)
+ subprocess.check_call(["hg", "clone", bot.remote, bot.local])
def popen_gen(cwd, args):
arg_str = map(str, args)
@@ 82,6 90,25 @@ def build_commit(local_path, commit_hash
status = "failed"
try:
+ build_phase("*** %s: %s ***" % (bot.longname, commit_hash))
+ build_phase("checkout")
+
+ revision = {
+ "revision": commit_hash
+ }
+
+ try:
+ hg("pull")
+ hg("update", "-C", "-r", commit_hash)
+ except:
+ print "Update failed, cloning from scratch..."
+ shutil.rmtree(bot.local, ignore_errors = True)
+ os.makedirs(bot.local)
+ hg("clone", bot.remote, bot.local)
+ hg("update", "-C", "-r", commit_hash)
+
+ hg("summary")
+
_,hg_branch = popen_str(local_path, ["hg", "id", "-b"])
_,hg_rev = popen_str(local_path, ["hg", "id", "-n"])
_,hg_tags = popen_str(local_path, ["hg", "id", "-t"])
@@ 92,21 119,6 @@ def build_commit(local_path, commit_hash
"tags": hg_tags.rstrip().split(" ")
}
- build_phase("*** %s: %s ***" % (bot.longname, commit_hash))
- build_phase("checkout")
- hg("pull")
-
- try:
- hg("update", "-C", "-r", commit_hash)
- except:
- print "Update failed, cloning from scratch..."
- shutil.rmtree(bot.local, ignore_errors = True)
- os.makedirs(bot.local)
- hg("clone", bot.remote, bot.local)
- hg("update", "-C", "-r", commit_hash)
-
- hg("summary")
-
try:
build_phase("configure")
cmake(*(bot.cmake_args + [local_path]))
@@ 153,7 165,7 @@ def toascii(s):
return s.encode("ascii") if isinstance(s, unicode) else s
-if bot.db_scheme is "https":
+if bot.db_scheme == "https":
couchdb = httplib.HTTPSConnection(bot.db_server)
else:
couchdb = httplib.HTTPConnection(bot.db_server)
@@ 170,7 182,7 @@ def request(method, selector = "/", body
headers.update(pheaders)
if not "Authorization" in headers:
- if not bot.passwd in [None, "NOT SET"]:
+ if not bot.passwd in [None, "NOT SET", ""]:
headers["Authorization"] = "Basic " + base64.b64encode(bot.user + ":" + bot.passwd)
@@ 178,6 190,7 @@ def request(method, selector = "/", body
body = json.dumps(body)
couchdb.request(method, toascii(selector), body, headers)
+
resp = couchdb.getresponse()
status = resp.status
rbody = resp.read()
@@ 206,20 219,34 @@ def range_query(start, end = None):
urllib.quote_plus(json.dumps(end)))
# delete any pending, incomplete builds by this agent
-def delete_by_view(view, startkey, endkey = None):
- query = "%s?%s&reduce=false" % (view, range_query(startkey, endkey))
- docs = GET("/_design/ci/_view/%s" % query)["rows"]
+def delete_by_view(view, startkey, endkey = None, descending = False, skip = None):
+ query = "%s?reduce=false" % view
+
+ if descending:
+ query = query + "&" + "descending=true"
+
+ if startkey:
+ query = query + "&" + range_query(startkey, endkey)
+
+ if skip:
+ query = query + "&" + "skip=%i" % skip
+
+ ids = [row["id"] for row in GET("/_design/ci/_view/%s" % query)["rows"]]
+
+ docs = POST("/_all_docs", { "keys": ids})["rows"]
+
update = {
- "docs": [{"_id": doc["id"], "_rev": doc["value"], "_deleted": True}
+ "docs": [{"_id": doc["id"], "_rev": doc["value"]["rev"], "_deleted": True}
for doc in docs
]
}
+ print update
return POST("/_bulk_docs", update)
# create database if not present
if "error" in GET(""):
- print "Creating database " + bot.couchdb + ":" + bot.database
+ print "Creating database " + bot.database
PUT("", None)
# remove my own aborted builds
@@ 227,91 254,101 @@ aborted = delete_by_view("pending_builds
for build in aborted:
print "Cleaning up aborted build %s: %s" % (build["id"], build["ok"])
-while True:
- # gather branch status
- branch_status = {}
+def prune_branch(botkey, branch):
+ delete_by_view("builds_gc", [botkey, branch], [botkey, branch, {}], True,)
+
+if __name__ == "__main__":
+ while True:
+ # gather branch status
+ branch_status = {}
- print "Checking branch status..."
- popen_str(bot.local, ["hg", "pull -r 300"])
- popen_str(bot.local, ["hg", "pull -r 600"])
- popen_str(bot.local, ["hg", "pull -r 900"])
- popen_str(bot.local, ["hg", "pull"])
+ print "Checking branch status..."
+ popen_str(bot.local, ["hg", "pull", "-r", "500"])
+ popen_str(bot.local, ["hg", "pull", "-r", "1000"])
+ popen_str(bot.local, ["hg", "pull", "-r", "1500"])
+ popen_str(bot.local, ["hg", "pull"])
+ branch_data = popen_gen(bot.local, ["hg", "branches", "-a", "--template", "{branch} {node}\\n"])
- branch_status = dict(
- branch.rsplit(None,1)
- for branch in popen_gen(bot.local, ["hg", "branches", "--template", "{branch} {node}\\n"]))
+ branch_status = dict(
+ branch.rsplit(None,1)
+ for branch in branch_data)
- for tag in popen_gen(bot.local, ["hg", "tags", "--template", "{tags} {node}\\n"]):
- data = tag.rsplit(None,1)
- if data[0].find("release") >= 0:
- branch_status[data[0]] = data[1]
+ for tag in itertools.islice(popen_gen(bot.local, ["hg", "tags", "--template", "{tags} {node}\\n"]),3):
+ data = tag.rsplit(None,1)
+ if data[0].find("release") >= 0:
+ branch_status[data[0]] = data[1]
- # figure out what to build
- branch_keys = urllib.quote_plus(json.dumps([[commit, bot.key] for commit in branch_status.values()]))
- have_builds = [row["key"][0] for row in GET("/_design/ci/_view/builds?keys=%s" % branch_keys)["rows"]]
+ # figure out what to build
+ branch_keys = urllib.quote_plus(json.dumps([[commit, bot.key] for commit in branch_status.values()]))
+ have_builds = [row["key"][0] for row in GET("/_design/ci/_view/builds?keys=%s" % branch_keys)["rows"]]
- for branch, commit in branch_status.iteritems():
-
- do_test = re.search(bot.test_branch, branch)
- do_package = re.search(bot.release_branch, branch)
+ for branch, commit in branch_status.iteritems():
+
+ do_test = re.search(bot.test_branch, branch)
+ do_package = re.search(bot.release_branch, branch)
- if do_test or do_package:
- if not commit in have_builds:
- workunit = {
- "type": "build",
- "status": "pending",
- "datetime": str(datetime.datetime.utcnow()),
- "agent": bot.name,
- "group": bot.key,
- "uname": platform.uname(),
- "commit": commit
- }
+ if do_test or do_package:
+ if not commit in have_builds:
+ workunit = {
+ "type": "build",
+ "status": "pending",
+ "datetime": datetime.datetime.utcnow().isoformat(),
+ "agent": bot.name,
+ "group": bot.key,
+ "uname": platform.uname(),
+ "revision": {
+ "branch": branch,
+ "revision": commit
+ },
+ "commit": commit
+ }
- # claim build
- identity = POST("/", workunit)
+ # claim build
+ identity = POST("/", workunit)
- if "ok" in identity:
- # claim successful, run the build
- print "Claimed branch: [%s]\n" % branch
+ if "ok" in identity:
+ # claim successful, run the build
+ print "Claimed branch: [%s] %s\n" % (branch, commit)
- # clean up leftover artifacts
- for globmask in bot.artifacts:
- for artifact in glob.glob("%s/%s" % (bot.workspace, globmask)):
- os.remove(artifact)
+ # clean up leftover artifacts
+ for globmask in bot.artifacts:
+ for artifact in glob.glob("%s/%s" % (bot.workspace, globmask)):
+ os.remove(artifact)
- # build
- workunit.update(_id = identity["id"], _rev = identity["rev"])
- workunit.update(build_commit(bot.local, workunit["commit"], do_package))
-
- # update build status document
- identity = POST("/", workunit)
+ # build
+ workunit.update(_id = identity["id"], _rev = identity["rev"])
+ workunit.update(build_commit(bot.local, workunit["commit"], do_package))
+
+ # update build status document
+ identity = POST("/", workunit)
- # glob artifacts and upload
- for globmask in bot.artifacts:
- for artifact in glob.glob("%s/%s" % (bot.workspace, globmask)):
- print "Posting " + artifact
- if artifact.endswith(".json"):
- # post as couchdb document
- print POST("/", json.loads(
- io.open(artifact, encoding = "utf-8", errors="ignore").read()))
- else:
- # post as a binary blob attachment
- print PUT("/%s/%s?rev=%s" % (identity["id"], os.path.basename(artifact), identity["rev"]),
- open(artifact, "rb").read(),
- { "Content-Type": "application/octet-stream" })
+ # glob artifacts and upload
+ for globmask in bot.artifacts:
+ for artifact in glob.glob("%s/%s" % (bot.workspace, globmask)):
+ print "Posting " + artifact
+ if artifact.endswith(".json"):
+ # post as couchdb document
+ print POST("/", json.loads(
+ io.open(artifact, encoding = "utf-8", errors="ignore").read()))
+ else:
+ # post as a binary blob attachment
+ print PUT("/%s/%s?rev=%s" % (identity["id"], os.path.basename(artifact), identity["rev"]),
+ open(artifact, "rb").read(),
+ { "Content-Type": "application/octet-stream" })
- # we completed a build, refresh work queue
- break
- else:
- print "Could not claim build: %s" % identityls
+ # we completed a build, refresh work queue
+ break
+ else:
+ print "Could not claim build: %s" % identity
+ else:
+ time.sleep(60)
- if bot.self_update is not None:
- proc = subprocess.Popen(["hg", "incoming", "-R", bot.self_update], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
- if proc.wait() is not 1:
- print "Updating couchbot..."
- break
+ if bot.self_update is not None:
+ proc = subprocess.Popen(["hg", "incoming", "-R", bot.self_update], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
+ if proc.wait() is not 1:
+ print "Updating couchbot..."
+ break
- time.sleep(60)