|
- #!/usr/bin/env ruby
- #
- ## Update data/devuan_packages.yml
-
- require 'fileutils'
- require 'net/https'
- require 'uri'
- require 'yaml'
- require 'xz'
-
- BASE = File.expand_path(File.dirname(File.dirname(__FILE__)))
-
- module Devuan
-
- module Constants
- PACKAGES_HOST = "packages.devuan.org"
- DISTS_PATH = "merged/dists"
- RELEASES = {
- stable: :jessie,
- testing: :ascii,
- unstable: :ceres
- }.freeze
- end
-
- module Helpers
-
- include Devuan::Constants
-
- def tmp_path(path = '')
- path = path.gsub('/', '-').gsub(/[^\w\.-]+/, '').gsub(/[-]+/, '-').strip
- [ BASE, 'tmp', path ].compact.join('/')
- end
-
- def packages_url(path = '')
- suite = RELEASES[@suite] || RELEASES[:stable]
- URI.parse("https://#{PACKAGES_HOST}/#{DISTS_PATH}/#{suite}/#{path}")
- end
-
- def wget(url)
- http = wconnect(url)
- http.request(Net::HTTP::Get.new(url.request_uri))
- end
-
- def wconnect(url)
- http = Net::HTTP.new(url.host, url.port)
- http.use_ssl = true
- http.verify_mode = OpenSSL::SSL::VERIFY_PEER
- http
- end
-
- end
-
- class WebFile
-
- include ::Devuan::Helpers
-
- attr_reader :data
-
- def initialize(options = {})
- @options = default_options.merge(options)
- @suite = options[:suite] || RELEASES[:stable]
- @uri = packages_url(@options[:uri])
- @copy = tmp_path(@options[:uri])
-
- # pass: `fetch: false` to use existing copy
- @options[:fetch] = options.key?(:fetch) ? (options[:fetch] === true) : true
-
- @data = {}
-
- _ensure_tmp_dir
- end
-
- def uri=(uri)
- @options[:uri] = uri
- @options[:fetch] = true
- @uri = packages_url(@options[:uri])
- @copy = tmp_path(@options[:uri])
- end
-
- def fetch
- if @options[:fetch]
- res = wget(@uri)
- File.open(@copy, 'w+') { |f| f.puts res.body }
- end
- if @copy =~ /.xz$/
- _decompress_xz(@copy) if @options[:fetch]
- @copy.gsub!(/.xz$/, '')
- end
- end
-
- # Override in inherited class, return @data
- def parse
- lines = File.readlines(@copy)
- raise NotImplementedError
- end
-
- protected
-
- def method_missing(*args)
- return @data[args.first] if @data.key?(args.first)
- super
- end
-
- def value_at(line)
- line.split(':').last.strip
- end
-
- private
-
- def default_options
- { uri: 'Release' }
- end
-
- def _decompress_xz(file)
- XZ.decompress_file(file, file.gsub(/.xz$/, ''))
- end
-
- def _ensure_tmp_dir()
- FileUtils.mkdir_p(tmp_path, mode: 1700)
- end
-
- end
-
- class GlobalReleaseFile < WebFile
- def parse
- lines = File.readlines(@copy)
- @data[:origin] = value_at(lines[0])
- @data[:label] = value_at(lines[1])
- @data[:suite] = value_at(lines[2])
- @data[:version] = value_at(lines[3])
- @data[:codename] = value_at(lines[4])
- @data[:valid_from] = value_at(lines[5])
- @data[:valid_until] = value_at(lines[6])
- @data[:architectures] = value_at(lines[7]).split(' ')
- @data[:components] = value_at(lines[8]).split(' ')
- @data[:description] = value_at(lines[9])
- # Line 10 is the "MD5Sum" title
- unless lines[10] =~ /^MD5Sum:$/
- $stderr.puts "Changing times! MD5Sum line not found, but: '#{lines[10].strip}'"
- return
- end
- # Grab all the MD5sums for verification
- lines.shift 11
- sums = lines.inject([]) do |x, l|
- s, z, f = l.strip.split(/\s+/, 3)
- x << { sum: s, size: z, file: f }
- x
- end
- @data[:sums] = sums.sort_by { |x| x[:file] } rescue sums
- @data
- end
- end
-
- class ArchReleaseFile < WebFile
- def parse
- lines = File.readlines(@copy)
- @data[:archive] = value_at(lines[0])
- @data[:origin] = value_at(lines[1])
- @data[:label] = value_at(lines[2])
- @data[:version] = value_at(lines[3])
- @data[:component] = value_at(lines[4])
- @data[:architecture] = value_at(lines[5])
- @data
- end
- end
-
- class ArchPackagesFile < WebFile
- def count
- @data[:packages].size
- end
-
- def current_package
- @data[:packages][@cursor]
- end
-
- def each(&block)
- @data[:packages].each(&block)
- end
-
- def select(&block)
- @data[:packages].select(&block)
- end
-
- def first
- @cursor = 0
- current_package
- end
-
- def next
- @cursor += 1
- current_package
- end
-
- def previous
- @cursor -= 1
- current_package
- end
-
- def last
- @cursor = count
- current_package
- end
-
- def parse
- lines = File.read(@copy)
- pkgs = lines.split("\n\n")
- @data[:packages] = []
- pkgs.each do |pkg|
- x = {}
- Devuan::Packages::Fields.each do |field|
- key = field.downcase.gsub('-', '_').intern
- if Devuan::Packages.stupid_field?(field)
- x[:stupid] ||= []
- x[:stupid] << field
- end
- if Devuan::Packages.multiline_field?(field)
- x[key] = value_for(field, pkg, :multiline => true)
- else
- x[key] = value_for(field, pkg)
- end
- end
- @data[:packages] << package_ds(x)
- end
- reset_cursor!
- @data
- end
-
- protected
-
- def reset_cursor!
- @cursor = 0
- end
-
- def value_for(field, string, options = {})
- if options[:multiline] # cut to next Field:
- $1 if string =~ /^#{field}: (.*?)(\n[A-Z][\w-]+:)/m
- else
- $1 if string =~ /^#{field}: (.*)$/
- end
- end
-
- protected
-
- # Turn raw data from Packages into a proper data structure
- def package_ds(raw)
- pkg = {
- name: raw[:package],
- provides: raw[:provides],
- description: raw[:description],
- maintainer: raw[:maintainer],
- essential: raw[:essential],
- origin: raw[:origin],
- priority: raw[:priority],
- section: raw[:section],
- task: raw[:task],
- popcon: popcon_for(raw[:package]),
- download_size: raw[:size],
- installed_size: raw[:installed_size],
- architectures: raw[:architectures],
- multi_arch: raw[:multi_arch],
- sums: {
- md5: raw[:md5sum],
- sha1: raw[:sha1],
- sha256: raw[:sha256]
- },
- links: {
- avatar: avatar_for(raw[:package]),
- homepage: raw[:homepage],
- issues: "https://git.devuan.org/devuan-packages/#{raw[:package]}/issues",
- milestones: "https://git.devuan.org/devuan-packages/#{raw[:package]}/milestones"
- },
- upstream: upstream_info_for(raw[:package]),
- dependencies: {
- breaks: raw[:breaks],
- conflicts: raw[:conflicts],
- depends: raw[:depends],
- pre_depends: raw[:pre_depends],
- recommends: raw[:recommends],
- suggests: raw[:suggests]
- },
- releases: {
- stable: {
- filename: "https://packages.devuan.org/#{raw[:filename]}",
- tarball: nil,
- version: raw[:version]
- },
- testing: {
- filename: "https://packages.devuan.org/#{raw[:filename]}",
- tarball: nil,
- version: raw[:version]
- },
- unstable: {
- filename: "https://packages.devuan.org/#{raw[:filename]}",
- tarball: nil,
- version: raw[:version]
- }
- },
- meta: {
- broken_home: raw[:homepage].nil?,
- stupid: !raw[:stupid].empty?,
- stupid_fields: raw[:stupid]
- }
- }
- end
-
- # Get avatar from gitlab
- def avatar_for(pkg)
- nil
- end
-
- # Get popcon info for package
- def popcon_for(pkg)
- nil
- end
-
- # Get upstream info for package
- def upstream_info_for(pkg)
- upstream = {
- homepage: nil,
- issues: nil,
- maintainer: nil,
- version: nil
- }
- end
- end
-
- module Packages
-
- # grep -e '^[^ :]*: ' tmp/main-binary-amd64-Packages | cut -d: -f1 | sort | uniq
- Fields = %w(Apport Architecture Breaks Bugs Build-Essential
- Built-Using Conflicts Depends Description Description-md5 Enhances
- Essential Filename Ghc-Package Gstreamer-Decoders Gstreamer-Elements
- Gstreamer-Encoders Gstreamer-Uri-Sinks Gstreamer-Uri-Sources
- Gstreamer-Version Homepage Installed-Size Lua-Versions Maintainer
- MD5sum Modaliases Multi-Arch Npp-Applications Npp-Description
- Npp-File Npp-Mimetype Npp-Name Origin Original-Maintainer
- Original-Source-Maintainer Package Package-Type Postgresql-Version
- Pre-Depends Priority Provides Python-Version Python-Versions
- Recommends Replaces Ruby-Versions Section SHA1 SHA256 Size Source
- Suggests Tag Task Version Xul-Appid)
-
- # Multiline fields
- MFields = %w(Description)
-
- # Stupid fields
- SFields = %w(Ghc-Package Gstreamer-Decoders Gstreamer-Elements
- Gstreamer-Encoders Gstreamer-Uri-Sinks Gstreamer-Uri-Sources
- Gstreamer-Version Lua-Versions Npp-Applications Npp-Description
- Npp-File Npp-Mimetype Npp-Name Postgresql-Version Python-Version
- Python-Versions Ruby-Versions Xul-Appid)
-
- class << self
-
- def multiline_field?(field)
- MFields.include?(field)
- end
-
- def stupid_field?(field)
- SFields.include?(field)
- end
-
- def release
- return @release unless @release.nil?
-
- @release = Devuan::GlobalReleaseFile.new
- @release.fetch
- @release.parse
- @release
- end
-
- def packages
- return @packages unless @packages.nil?
-
- @packages = Devuan::ArchPackagesFile.new(uri: "main/binary-amd64/Packages.xz", fetch: false)
- @packages.fetch
- @packages.parse
- @packages
- end
-
- # TODO: ask amprolla instead
- def blacklisted
- [ 'systemd', 'systemd-sysvinit' ]
- end
-
- def named(name)
- packages.select { |x| x[:package] =~ /#{name}/ }
- end
-
- def depending_on(package)
- pattern = Regexp.new(package)
- packages.select do |x|
- x[:pre_depends] =~ pattern || x[:depends] =~ pattern
- end
- end
-
- def from_devuan
- packages.select { |x| x[:releases][:stable][:version] =~ /[+]devuan/ }
- end
-
- def save_to_yml
- File.open(BASE + '/tmp/packages.yml', 'w+') do |f|
- f.puts(YAML.dump(packages))
- end
- File.open(BASE + '/tmp/devuan_packages.yml', 'w+') do |f|
- f.puts(YAML.dump(from_devuan))
- end
- File.open(BASE + '/tmp/suspicious_packages.yml', 'w+') do |f|
- f.puts(YAML.dump(depending_on('libsystemd0')))
- end
- end
-
- protected
-
- def method_missing(*args)
- if packages.respond_to?(args.first)
- packages.send(args.shift, *args)
- else
- super
- end
- end
- end # class << self
- end # Module Packages
- end # Module Devuan
-
-
- release = Devuan::GlobalReleaseFile.new(fetch: false)
- release.fetch
- release.parse
-
- puts "Origin: #{release.origin}, version #{release.version}"
- puts "Architectures: #{release.architectures}"
- puts "Retrieving package information... Might take a while..."
- puts "Retrieved information about #{Devuan::Packages.count} packages."
- puts "Among those, #{Devuan::Packages.from_devuan.count} packages were modified by Devuan."
-
- puts "E.g., #{Devuan::Packages.from_devuan.first[:name]}..."
-
- puts "Saving to YAML format"
- Devuan::Packages.save_to_yml
-
- puts "Here comes the list of devuan-modified packages..."
- puts Devuan::Packages.from_devuan.map { |x| x[:name] }.join(', ')
-
- puts "Here is a list of packages starting with devuan:"
- p Devuan::Packages.named('^devuan')
- #.map { |x| x[:name] }.join(', ')
-
- puts "#{Devuan::Packages.named('systemd').count} packages contain `systemd` in their name."
-
- # Suspicious packages
- libsystemd0_dependent = Devuan::Packages.depending_on('libsystemd0')
- systemd_dependent = Devuan::Packages.depending_on('systemd')
- remaining_packages_to_fix = libsystemd0_dependent - systemd_dependent
- count = remaining_packages_to_fix.count
-
- puts "There are #{libsystemd0_dependent.count} packages depending on libsystemd0:"
- p libsystemd0_dependent.class
- puts libsystemd0_dependent.map { |e| e[:name] }.join(', ')
-
- p libsystemd0_dependent.first
-
- if count > 0
- puts "There are #{count} packages with dependency on systemd."
- puts remaining_packages_to_fix.map(&:name).join(', ') unless count > 42
- else
- puts "Devuan #{release.version} is free from systemd dependencies!"
- end
|