Devuan fork of gpsd
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

357 lines
11 KiB

  1. #!/usr/bin/env python
  2. #
  3. # This file is Copyright (c) 2010 by the GPSD project
  4. # BSD terms apply: see the file COPYING in the distribution root for details.
  5. #
  6. """\
  7. flocktest - shepherd script for the GPSD test flock
  8. usage: flocktest [-c] [-q] [-d subdir] [-k key] -v [-x exclude] [-?]
  9. The -? makes flocktest prints this help and exits.
  10. The -c option dumps flocktest's configuration and exits
  11. The -k mode installs a specified file of ssh public keys on all machines
  12. Otherwise, the remote flockdriver script is executed on each machine.
  13. The -d option passes it a name for the remote test subdirectory
  14. If you do not specify a subdirectory name, the value of $LOGNAME will be used.
  15. The -q option suppresses CIA notifications
  16. The -v option shows all ssh commands issued, runs flockdriver with -x set
  17. and causes logs to be echoed even on success.
  18. The -x option specifies a comma-separated list of items that are
  19. either remote hostnames or architecture tags. Matching sites are
  20. excluded. You may wish to use this to avoid doing remote tests that
  21. are redundant with your local ones.
  22. Known bug: The -k has no atomicity check. Running it from two
  23. flocktest instances concurrently could result in a scrambled keyfile.
  24. """
  25. # This code runs compatibly under Python 2 and 3.x for x >= 2.
  26. # Preserve this property!
  27. from __future__ import absolute_import, print_function, division
  28. import os, sys, getopt, socket, threading, time
  29. try:
  30. import configparser # Python 2
  31. except ImportError:
  32. import ConfigParser as configparser # Python 3
  33. try:
  34. import commands # Python 2
  35. except ImportError:
  36. import subprocess as commands # Python 3
  37. flockdriver = '''
  38. #!/bin/sh
  39. #
  40. # flockdriver - conduct regression tests as an agent for a remote flocktest
  41. #
  42. # This file was generated at %(date)s. Do not hand-hack.
  43. quiet=no
  44. while getopts dq opt
  45. do
  46. case $opt in
  47. d) subdir=$2; shift; shift ;;
  48. q) quiet=yes; shift ;;
  49. esac
  50. done
  51. # Fully qualified domain name of the repo host. You can hardwire this
  52. # to make the script faster. The -f option works under Linux and FreeBSD,
  53. # but not OpenBSD and NetBSD. But under OpenBSD and NetBSD,
  54. # hostname without options gives the FQDN.
  55. if hostname -f >/dev/null 2>&1
  56. then
  57. site=`hostname -f`
  58. else
  59. site=`hostname`
  60. fi
  61. if [ -f "flockdriver.lock" ]
  62. then
  63. logmessage="A test was already running when you initiated this one."
  64. cd $subdir
  65. else
  66. echo "Test begins: "`date`
  67. echo "Site: $site"
  68. echo "Directory: ${PWD}/${subdir}"
  69. # Check the origin against the repo origin. If they do not match,
  70. # force a re-clone of the repo
  71. if [ -d $subdir ]
  72. then
  73. repo_origin=`(cd $subdir; git config remote.origin.url)`
  74. if [ $repo_origin != "%(origin)s" ]
  75. then
  76. echo "Forced re-clone."
  77. rm -fr $subdir
  78. fi
  79. fi
  80. # Set up or update the repo
  81. if [ ! -d $subdir ]
  82. then
  83. git clone %(origin)s $subdir
  84. cd $subdir
  85. else
  86. cd $subdir;
  87. git pull
  88. fi
  89. # Scripts in the test directory need to be able to run binaries in same
  90. PATH="$PATH:."
  91. # Perform the test
  92. if ( %(regression)s )
  93. then
  94. logmessage="Regression test succeeded."
  95. status=0
  96. else
  97. logmessage="Regression test failed."
  98. status=1
  99. fi
  100. echo "Test ends: "`date`
  101. fi
  102. # Here is where we abuse CIA to do our notifications for us.
  103. # Addresses for the e-mail
  104. from="FLOCKDRIVER-NOREPLY@${site}"
  105. to="cia@cia.navi.cx"
  106. # SMTP client to use
  107. sendmail="sendmail -t -f ${from}"
  108. # Should include all places sendmail is likely to lurk.
  109. PATH="$PATH:/usr/sbin/"
  110. # Identify what just succeeded or failed
  111. merged=$(git rev-parse HEAD)
  112. rev=$(git describe ${merged} 2>/dev/null)
  113. [ -z ${rev} ] && rev=${merged}
  114. refname=$(git symbolic-ref HEAD 2>/dev/null)
  115. refname=${refname##refs/heads/}
  116. # And the git version
  117. gitver=$(git --version)
  118. gitver=${gitver##* }
  119. if [ $quiet = no ]
  120. then
  121. ${sendmail} << EOM
  122. Message-ID: <${merged}.${subdir}.blip@%(project)s>
  123. From: ${from}
  124. To: ${to}
  125. Content-type: text/xml
  126. Subject: DeliverXML
  127. <message>
  128. <generator>
  129. <name>%(project)s Remote Test Flock Driver</name>
  130. <version>${gitver}</version>
  131. <url>${origin}/flockdriver</url>
  132. </generator>
  133. <source>
  134. <project>%(project)s</project>
  135. <branch>${refname}@${site}</branch>
  136. </source>
  137. <timestamp>`date`</timestamp>
  138. <body>
  139. <commit>
  140. <author>${subdir}</author>
  141. <revision>${rev}</revision>
  142. <log>${logmessage}</log>
  143. </commit>
  144. </body>
  145. </message>
  146. EOM
  147. fi
  148. exit $status
  149. # End.
  150. '''
  151. class FlockThread(threading.Thread):
  152. def __init__(self, site, command):
  153. threading.Thread.__init__(self)
  154. self.site = site
  155. self.command = command
  156. def run(self):
  157. (self.status, self.output) = commands.getstatusoutput(self.command)
  158. class TestSite(object):
  159. "Methods for performing tests on a single remote site."
  160. def __init__(self, fqdn, config, execute=True):
  161. self.fqdn = fqdn
  162. self.config = config
  163. self.execute = execute
  164. self.me = self.config["login"] + "@" + self.fqdn
  165. def error(self, msg):
  166. "Report an error while executing a remote command."
  167. sys.stderr.write("%s: %s\n" % (self.fqdn, msg))
  168. def do_remote(self, remote):
  169. "Execute a command on a specified remote host."
  170. command = "ssh "
  171. if "port" in self.config:
  172. command += "-p %s " % self.config["port"]
  173. command += "%s '%s'" % (self.me, remote)
  174. if self.verbose:
  175. print(command)
  176. self.thread = FlockThread(self, command)
  177. self.thread.start()
  178. def update_keys(self, filename):
  179. "Upload a specified file to replace the remote authorized keys."
  180. if 'debian.org' in self.me:
  181. self.error("updating keys on debian.org machines makes no sense.")
  182. return 1
  183. command = "scp '%s' %s:~/.ssh/.authorized_keys" % (os.path.expanduser(filename), self.me)
  184. if self.verbose:
  185. print(command)
  186. status = os.system(command)
  187. if status:
  188. self.error("copy with '%s' failed" % command)
  189. return status
  190. def do_append(self, filename, string):
  191. "Append a line to a specified remote file, in foreground."
  192. self.do_remote("echo \"%s\" >>%s" % (string.strip(), filename))
  193. def do_flockdriver(self, agent, invocation):
  194. "Copy flockdriver to the remote site and run it."
  195. self.starttime = time.time()
  196. if self.config.get("quiet", "no") == "yes":
  197. invocation += " -q"
  198. uploader = "ssh -p %s %s 'cat >%s'" \
  199. % (self.config.get("port", "22"), self.me, agent)
  200. if self.verbose:
  201. print(uploader)
  202. ofp = os.popen(uploader, "w")
  203. self.config['date'] = time.ctime()
  204. ofp.write(flockdriver % self.config)
  205. if ofp.close():
  206. print("flocktest: agent upload failed", file=sys.stderr)
  207. else:
  208. self.do_remote(invocation)
  209. self.elapsed = time.time() - self.starttime
  210. class TestFlock(object):
  211. "Methods for performing parallel tests on a flock of remote sites."
  212. ssh_options = "no-port-forwarding,no-X11-forwarding," \
  213. "no-agent-forwarding,no-pty "
  214. def __init__(self, sitelist, verbose=False):
  215. self.sitelist = sitelist
  216. self.verbose = verbose
  217. def update_remote(self, filename):
  218. "Copy a specified file to the remote home on all machines."
  219. for site in self.sitelist:
  220. site.update_remote(filename)
  221. def do_remotes(self, agent, invocation):
  222. "Execute a command on all machines in the flock."
  223. slaves = []
  224. print("== testing at: %s ==" % flock.listdump())
  225. starttime = time.time()
  226. for site in self.sitelist:
  227. site.do_flockdriver(agent, invocation)
  228. for site in sites:
  229. site.thread.join()
  230. failed = 0
  231. for site in sites:
  232. if site.thread.status:
  233. print("== %s test FAILED in %.2f seconds, status %d ==" % (site.fqdn, site.elapsed, site.thread.status))
  234. failed += 1
  235. print(site.thread.output)
  236. else:
  237. print("== %s test succeeded in %.2f seconds ==" % (site.fqdn, site.elapsed))
  238. if self.verbose:
  239. print(site.thread.output)
  240. elapsed = time.time() - starttime
  241. print("== %d tests completed in %.2f seconds: %d failed ==" % (len(sites), elapsed, failed))
  242. def exclude(self, exclusions):
  243. "Delete matching sites."
  244. self.sitelist = [x for x in self.sitelist if x.fqdn not in exclusions and x.config["arch"] not in exclusions]
  245. def update_keys(self, keyfile):
  246. "Copy the specified public key file to all sites."
  247. for site in self.sitelist:
  248. site.update_keys(keyfile)
  249. def listdump(self):
  250. "Return a dump of the site list."
  251. return ", ".join([x.fqdn for x in self.sitelist])
  252. if __name__ == '__main__':
  253. try:
  254. (options, arguments) = getopt.getopt(sys.argv[1:], "cd:kqvx:?")
  255. except getopt.GetoptError as msg:
  256. print("flocktest: " + str(msg))
  257. raise SystemExit(1)
  258. exclusions = []
  259. subdir = None
  260. copykeys = None
  261. verbose = False
  262. dumpconf = False
  263. cianotify = True
  264. for (switch, val) in options:
  265. if switch == '-c': # Dump flocktest configuration
  266. dumpconf = True
  267. elif switch == '-d': # Set the test subdirectory name
  268. subdir = val
  269. elif switch == '-k': # Install the access keys
  270. copykeys = True
  271. elif switch == '-q': # Suppress CIA notifications
  272. cianotify = False
  273. elif switch == '-v': # Display build log even when no error
  274. verbose = True
  275. elif switch == '-x': # Exclude specified sites or architectures
  276. exclusions = [x.strip() for x in val.split(",")]
  277. else: # switch == '-?':
  278. print(__doc__)
  279. sys.exit(0)
  280. config = configparser.RawConfigParser()
  281. config.read(["flocktest.ini", ".flocktest.ini"])
  282. if arguments:
  283. config.set("DEFAULT", "origin", arguments[0])
  284. if not config.has_option("DEFAULT", "origin"):
  285. print("flocktest: repository required.", file=sys.stderr)
  286. sys.exit(1)
  287. sites = []
  288. for site in config.sections():
  289. newsite = TestSite(site, dict(config.items(site)))
  290. newsite.verbose = verbose
  291. if newsite.config["status"].lower() == "up":
  292. sites.append(newsite)
  293. flock = TestFlock(sites, verbose)
  294. if exclusions:
  295. flock.exclude(exclusions)
  296. if dumpconf:
  297. config.write(sys.stdout)
  298. elif copykeys:
  299. keyfile = config.get("DEFAULT", "sshkeys")
  300. flock.update_keys(keyfile)
  301. else:
  302. if not subdir:
  303. subdir = os.getenv("LOGNAME")
  304. if not subdir:
  305. print("flocktest: you don't exist, go away!")
  306. sys.exit(1)
  307. agent = "flockdriver.%s" % subdir
  308. invocation = "sh flockdriver.%s -d %s" % (subdir, subdir,)
  309. if not cianotify:
  310. invocation += " -q"
  311. if verbose > 1:
  312. invocation = "sh -x " + invocation
  313. flock.do_remotes(agent, invocation)
  314. # The following sets edit modes for GNU EMACS
  315. # Local Variables:
  316. # mode:python
  317. # End: