Main repository for Devuan's www.devuan.org.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

468 lines
12 KiB

  1. #!/usr/bin/env ruby
  2. #
  3. ## Update data/devuan_packages.yml
  4. require 'fileutils'
  5. require 'net/https'
  6. require 'uri'
  7. require 'yaml'
  8. require 'xz'
  9. BASE = File.expand_path(File.dirname(File.dirname(__FILE__)))
  10. module Devuan
  11. module Constants
  12. PACKAGES_HOST = "packages.devuan.org"
  13. DISTS_PATH = "merged/dists"
  14. RELEASES = {
  15. stable: :jessie,
  16. testing: :ascii,
  17. unstable: :ceres
  18. }.freeze
  19. end
  20. module Helpers
  21. include Devuan::Constants
  22. def tmp_path(path = '')
  23. path = path.gsub('/', '-').gsub(/[^\w\.-]+/, '').gsub(/[-]+/, '-').strip
  24. [ BASE, 'tmp', path ].compact.join('/')
  25. end
  26. def packages_url(path = '')
  27. suite = RELEASES[@suite] || RELEASES[:stable]
  28. URI.parse("https://#{PACKAGES_HOST}/#{DISTS_PATH}/#{suite}/#{path}")
  29. end
  30. def wget(url)
  31. http = wconnect(url)
  32. http.request(Net::HTTP::Get.new(url.request_uri))
  33. end
  34. def wconnect(url)
  35. http = Net::HTTP.new(url.host, url.port)
  36. http.use_ssl = true
  37. http.verify_mode = OpenSSL::SSL::VERIFY_PEER
  38. http
  39. end
  40. end
  41. class WebFile
  42. include ::Devuan::Helpers
  43. attr_reader :data
  44. def initialize(options = {})
  45. @options = default_options.merge(options)
  46. @suite = options[:suite] || RELEASES[:stable]
  47. @uri = packages_url(@options[:uri])
  48. @copy = tmp_path(@options[:uri])
  49. # pass: `fetch: false` to use existing copy
  50. @options[:fetch] = options.key?(:fetch) ? (options[:fetch] === true) : true
  51. @data = {}
  52. _ensure_tmp_dir
  53. end
  54. def uri=(uri)
  55. @options[:uri] = uri
  56. @options[:fetch] = true
  57. @uri = packages_url(@options[:uri])
  58. @copy = tmp_path(@options[:uri])
  59. end
  60. def fetch
  61. if @options[:fetch]
  62. res = wget(@uri)
  63. File.open(@copy, 'w+') { |f| f.puts res.body }
  64. end
  65. if @copy =~ /.xz$/
  66. _decompress_xz(@copy) if @options[:fetch]
  67. @copy.gsub!(/.xz$/, '')
  68. end
  69. end
  70. # Override in inherited class, return @data
  71. def parse
  72. lines = File.readlines(@copy)
  73. raise NotImplementedError
  74. end
  75. protected
  76. def method_missing(*args)
  77. return @data[args.first] if @data.key?(args.first)
  78. super
  79. end
  80. def value_at(line)
  81. line.split(':').last.strip
  82. end
  83. private
  84. def default_options
  85. { uri: 'Release' }
  86. end
  87. def _decompress_xz(file)
  88. XZ.decompress_file(file, file.gsub(/.xz$/, ''))
  89. end
  90. def _ensure_tmp_dir()
  91. FileUtils.mkdir_p(tmp_path, mode: 1700)
  92. end
  93. end
  94. class GlobalReleaseFile < WebFile
  95. def parse
  96. lines = File.readlines(@copy)
  97. @data[:origin] = value_at(lines[0])
  98. @data[:label] = value_at(lines[1])
  99. @data[:suite] = value_at(lines[2])
  100. @data[:version] = value_at(lines[3])
  101. @data[:codename] = value_at(lines[4])
  102. @data[:valid_from] = value_at(lines[5])
  103. @data[:valid_until] = value_at(lines[6])
  104. @data[:architectures] = value_at(lines[7]).split(' ')
  105. @data[:components] = value_at(lines[8]).split(' ')
  106. @data[:description] = value_at(lines[9])
  107. # Line 10 is the "MD5Sum" title
  108. unless lines[10] =~ /^MD5Sum:$/
  109. $stderr.puts "Changing times! MD5Sum line not found, but: '#{lines[10].strip}'"
  110. return
  111. end
  112. # Grab all the MD5sums for verification
  113. lines.shift 11
  114. sums = lines.inject([]) do |x, l|
  115. s, z, f = l.strip.split(/\s+/, 3)
  116. x << { sum: s, size: z, file: f }
  117. x
  118. end
  119. @data[:sums] = sums.sort_by { |x| x[:file] } rescue sums
  120. @data
  121. end
  122. end
  123. class ArchReleaseFile < WebFile
  124. def parse
  125. lines = File.readlines(@copy)
  126. @data[:archive] = value_at(lines[0])
  127. @data[:origin] = value_at(lines[1])
  128. @data[:label] = value_at(lines[2])
  129. @data[:version] = value_at(lines[3])
  130. @data[:component] = value_at(lines[4])
  131. @data[:architecture] = value_at(lines[5])
  132. @data
  133. end
  134. end
  135. class ArchPackagesFile < WebFile
  136. def count
  137. @data[:packages].size
  138. end
  139. def current_package
  140. @data[:packages][@cursor]
  141. end
  142. def each(&block)
  143. @data[:packages].each(&block)
  144. end
  145. def select(&block)
  146. @data[:packages].select(&block)
  147. end
  148. def first
  149. @cursor = 0
  150. current_package
  151. end
  152. def next
  153. @cursor += 1
  154. current_package
  155. end
  156. def previous
  157. @cursor -= 1
  158. current_package
  159. end
  160. def last
  161. @cursor = count
  162. current_package
  163. end
  164. def parse
  165. lines = File.read(@copy)
  166. pkgs = lines.split("\n\n")
  167. @data[:packages] = []
  168. pkgs.each do |pkg|
  169. x = {}
  170. Devuan::Packages::Fields.each do |field|
  171. key = field.downcase.gsub('-', '_').intern
  172. if Devuan::Packages.stupid_field?(field)
  173. x[:stupid] ||= []
  174. x[:stupid] << field
  175. end
  176. if Devuan::Packages.multiline_field?(field)
  177. x[key] = value_for(field, pkg, :multiline => true)
  178. else
  179. x[key] = value_for(field, pkg)
  180. end
  181. end
  182. @data[:packages] << package_ds(x)
  183. end
  184. reset_cursor!
  185. @data
  186. end
  187. protected
  188. def reset_cursor!
  189. @cursor = 0
  190. end
  191. def value_for(field, string, options = {})
  192. if options[:multiline] # cut to next Field:
  193. $1 if string =~ /^#{field}: (.*?)(\n[A-Z][\w-]+:)/m
  194. else
  195. $1 if string =~ /^#{field}: (.*)$/
  196. end
  197. end
  198. protected
  199. # Turn raw data from Packages into a proper data structure
  200. def package_ds(raw)
  201. pkg = {
  202. name: raw[:package],
  203. provides: raw[:provides],
  204. description: raw[:description],
  205. maintainer: raw[:maintainer],
  206. essential: raw[:essential],
  207. origin: raw[:origin],
  208. priority: raw[:priority],
  209. section: raw[:section],
  210. task: raw[:task],
  211. popcon: popcon_for(raw[:package]),
  212. download_size: raw[:size],
  213. installed_size: raw[:installed_size],
  214. architectures: raw[:architectures],
  215. multi_arch: raw[:multi_arch],
  216. sums: {
  217. md5: raw[:md5sum],
  218. sha1: raw[:sha1],
  219. sha256: raw[:sha256]
  220. },
  221. links: {
  222. avatar: avatar_for(raw[:package]),
  223. homepage: raw[:homepage],
  224. issues: "https://git.devuan.org/devuan-packages/#{raw[:package]}/issues",
  225. milestones: "https://git.devuan.org/devuan-packages/#{raw[:package]}/milestones"
  226. },
  227. upstream: upstream_info_for(raw[:package]),
  228. dependencies: {
  229. breaks: raw[:breaks],
  230. conflicts: raw[:conflicts],
  231. depends: raw[:depends],
  232. pre_depends: raw[:pre_depends],
  233. recommends: raw[:recommends],
  234. suggests: raw[:suggests]
  235. },
  236. releases: {
  237. stable: {
  238. filename: "https://packages.devuan.org/#{raw[:filename]}",
  239. tarball: nil,
  240. version: raw[:version]
  241. },
  242. testing: {
  243. filename: "https://packages.devuan.org/#{raw[:filename]}",
  244. tarball: nil,
  245. version: raw[:version]
  246. },
  247. unstable: {
  248. filename: "https://packages.devuan.org/#{raw[:filename]}",
  249. tarball: nil,
  250. version: raw[:version]
  251. }
  252. },
  253. meta: {
  254. broken_home: raw[:homepage].nil?,
  255. stupid: !raw[:stupid].empty?,
  256. stupid_fields: raw[:stupid]
  257. }
  258. }
  259. end
  260. # Get avatar from gitlab
  261. def avatar_for(pkg)
  262. nil
  263. end
  264. # Get popcon info for package
  265. def popcon_for(pkg)
  266. nil
  267. end
  268. # Get upstream info for package
  269. def upstream_info_for(pkg)
  270. upstream = {
  271. homepage: nil,
  272. issues: nil,
  273. maintainer: nil,
  274. version: nil
  275. }
  276. end
  277. end
  278. module Packages
  279. # grep -e '^[^ :]*: ' tmp/main-binary-amd64-Packages | cut -d: -f1 | sort | uniq
  280. Fields = %w(Apport Architecture Breaks Bugs Build-Essential
  281. Built-Using Conflicts Depends Description Description-md5 Enhances
  282. Essential Filename Ghc-Package Gstreamer-Decoders Gstreamer-Elements
  283. Gstreamer-Encoders Gstreamer-Uri-Sinks Gstreamer-Uri-Sources
  284. Gstreamer-Version Homepage Installed-Size Lua-Versions Maintainer
  285. MD5sum Modaliases Multi-Arch Npp-Applications Npp-Description
  286. Npp-File Npp-Mimetype Npp-Name Origin Original-Maintainer
  287. Original-Source-Maintainer Package Package-Type Postgresql-Version
  288. Pre-Depends Priority Provides Python-Version Python-Versions
  289. Recommends Replaces Ruby-Versions Section SHA1 SHA256 Size Source
  290. Suggests Tag Task Version Xul-Appid)
  291. # Multiline fields
  292. MFields = %w(Description)
  293. # Stupid fields
  294. SFields = %w(Ghc-Package Gstreamer-Decoders Gstreamer-Elements
  295. Gstreamer-Encoders Gstreamer-Uri-Sinks Gstreamer-Uri-Sources
  296. Gstreamer-Version Lua-Versions Npp-Applications Npp-Description
  297. Npp-File Npp-Mimetype Npp-Name Postgresql-Version Python-Version
  298. Python-Versions Ruby-Versions Xul-Appid)
  299. class << self
  300. def multiline_field?(field)
  301. MFields.include?(field)
  302. end
  303. def stupid_field?(field)
  304. SFields.include?(field)
  305. end
  306. def release
  307. return @release unless @release.nil?
  308. @release = Devuan::GlobalReleaseFile.new
  309. @release.fetch
  310. @release.parse
  311. @release
  312. end
  313. def packages
  314. return @packages unless @packages.nil?
  315. @packages = Devuan::ArchPackagesFile.new(uri: "main/binary-amd64/Packages.xz", fetch: false)
  316. @packages.fetch
  317. @packages.parse
  318. @packages
  319. end
  320. # TODO: ask amprolla instead
  321. def blacklisted
  322. [ 'systemd', 'systemd-sysvinit' ]
  323. end
  324. def named(name)
  325. packages.select { |x| x[:package] =~ /#{name}/ }
  326. end
  327. def depending_on(package)
  328. pattern = Regexp.new(package)
  329. packages.select do |x|
  330. x[:pre_depends] =~ pattern || x[:depends] =~ pattern
  331. end
  332. end
  333. def from_devuan
  334. packages.select { |x| x[:releases][:stable][:version] =~ /[+]devuan/ }
  335. end
  336. def save_to_yml
  337. File.open(BASE + '/tmp/packages.yml', 'w+') do |f|
  338. f.puts(YAML.dump(packages))
  339. end
  340. File.open(BASE + '/tmp/devuan_packages.yml', 'w+') do |f|
  341. f.puts(YAML.dump(from_devuan))
  342. end
  343. File.open(BASE + '/tmp/suspicious_packages.yml', 'w+') do |f|
  344. f.puts(YAML.dump(depending_on('libsystemd0')))
  345. end
  346. end
  347. protected
  348. def method_missing(*args)
  349. if packages.respond_to?(args.first)
  350. packages.send(args.shift, *args)
  351. else
  352. super
  353. end
  354. end
  355. end # class << self
  356. end # Module Packages
  357. end # Module Devuan
  358. release = Devuan::GlobalReleaseFile.new(fetch: false)
  359. release.fetch
  360. release.parse
  361. puts "Origin: #{release.origin}, version #{release.version}"
  362. puts "Architectures: #{release.architectures}"
  363. puts "Retrieving package information... Might take a while..."
  364. puts "Retrieved information about #{Devuan::Packages.count} packages."
  365. puts "Among those, #{Devuan::Packages.from_devuan.count} packages were modified by Devuan."
  366. puts "E.g., #{Devuan::Packages.from_devuan.first[:name]}..."
  367. puts "Saving to YAML format"
  368. Devuan::Packages.save_to_yml
  369. puts "Here comes the list of devuan-modified packages..."
  370. puts Devuan::Packages.from_devuan.map { |x| x[:name] }.join(', ')
  371. puts "Here is a list of packages starting with devuan:"
  372. p Devuan::Packages.named('^devuan')
  373. #.map { |x| x[:name] }.join(', ')
  374. puts "#{Devuan::Packages.named('systemd').count} packages contain `systemd` in their name."
  375. # Suspicious packages
  376. libsystemd0_dependent = Devuan::Packages.depending_on('libsystemd0')
  377. systemd_dependent = Devuan::Packages.depending_on('systemd')
  378. remaining_packages_to_fix = libsystemd0_dependent - systemd_dependent
  379. count = remaining_packages_to_fix.count
  380. puts "There are #{libsystemd0_dependent.count} packages depending on libsystemd0:"
  381. p libsystemd0_dependent.class
  382. puts libsystemd0_dependent.map { |e| e[:name] }.join(', ')
  383. p libsystemd0_dependent.first
  384. if count > 0
  385. puts "There are #{count} packages with dependency on systemd."
  386. puts remaining_packages_to_fix.map(&:name).join(', ') unless count > 42
  387. else
  388. puts "Devuan #{release.version} is free from systemd dependencies!"
  389. end