www.devuan.org/bin/update-packages

#!/usr/bin/env ruby
#
## Update data/devuan_packages.yml

require 'fileutils'
require 'net/https'
require 'uri'
require 'yaml'
require 'xz'

BASE = File.expand_path(File.dirname(File.dirname(__FILE__)))

module Devuan

  module Constants
    PACKAGES_HOST = "packages.devuan.org"
    DISTS_PATH    = "merged/dists"
    RELEASES      = {
      stable:   :jessie,
      testing:  :ascii,
      unstable: :ceres
    }.freeze
  end

  module Helpers

    include Devuan::Constants

    def tmp_path(path = '')
      path = path.gsub('/', '-').gsub(/[^\w\.-]+/, '').gsub(/[-]+/, '-').strip
      [ BASE, 'tmp', path ].compact.join('/')
    end

    def packages_url(path = '')
      suite = RELEASES[@suite] || RELEASES[:stable]
      URI.parse("https://#{PACKAGES_HOST}/#{DISTS_PATH}/#{suite}/#{path}")
    end

    def wget(url)
      http = wconnect(url)
      http.request(Net::HTTP::Get.new(url.request_uri))
    end

    def wconnect(url)
      http             = Net::HTTP.new(url.host, url.port)
      http.use_ssl     = true
      http.verify_mode = OpenSSL::SSL::VERIFY_PEER
      http
    end

  end

  class WebFile

    include ::Devuan::Helpers

    attr_reader :data

    def initialize(options = {})
      @options = default_options.merge(options)
      @suite   = options[:suite] || RELEASES[:stable]
      @uri     = packages_url(@options[:uri])
      @copy    = tmp_path(@options[:uri])

      # pass: `fetch: false` to use existing copy
      @options[:fetch] = options.key?(:fetch) ? (options[:fetch] === true) : true

      @data = {}

      _ensure_tmp_dir
    end

    def uri=(uri)
      @options[:uri]   = uri
      @options[:fetch] = true
      @uri             = packages_url(@options[:uri])
      @copy            = tmp_path(@options[:uri])
    end

    def fetch
      if @options[:fetch]
        res = wget(@uri)
        File.open(@copy, 'w+') { |f| f.puts res.body }
      end
      if @copy =~ /.xz$/
        _decompress_xz(@copy) if @options[:fetch]
        @copy.gsub!(/.xz$/, '')
      end
    end

    # Override in inherited class, return @data
    def parse
      lines = File.readlines(@copy)
      raise NotImplementedError
    end

    protected

    def method_missing(*args)
      return @data[args.first] if @data.key?(args.first)
      super
    end

    def value_at(line)
      line.split(':').last.strip
    end

    private

    def default_options
      { uri: 'Release' }
    end

    def _decompress_xz(file)
      XZ.decompress_file(file, file.gsub(/.xz$/, ''))
    end

    def _ensure_tmp_dir()
      FileUtils.mkdir_p(tmp_path, mode: 1700)
    end

  end

  class GlobalReleaseFile < WebFile
    def parse
      lines = File.readlines(@copy)
      @data[:origin]        = value_at(lines[0])
      @data[:label]         = value_at(lines[1])
      @data[:suite]         = value_at(lines[2])
      @data[:version]       = value_at(lines[3])
      @data[:codename]      = value_at(lines[4])
      @data[:valid_from]    = value_at(lines[5])
      @data[:valid_until]   = value_at(lines[6])
      @data[:architectures] = value_at(lines[7]).split(' ')
      @data[:components]    = value_at(lines[8]).split(' ')
      @data[:description]   = value_at(lines[9])
      # Line 10 is the "MD5Sum" title
      unless lines[10] =~ /^MD5Sum:$/
        $stderr.puts "Changing times!  MD5Sum line not found, but: '#{lines[10].strip}'"
        return
      end
      # Grab all the MD5sums for verification
      lines.shift 11
      sums = lines.inject([]) do |x, l|
        s, z, f = l.strip.split(/\s+/, 3)
        x << { sum: s, size: z, file: f }
        x
      end
      @data[:sums] = sums.sort_by { |x| x[:file] } rescue sums
      @data
    end
  end

  class ArchReleaseFile < WebFile
    def parse
      lines = File.readlines(@copy)
      @data[:archive]       = value_at(lines[0])
      @data[:origin]        = value_at(lines[1])
      @data[:label]         = value_at(lines[2])
      @data[:version]       = value_at(lines[3])
      @data[:component]     = value_at(lines[4])
      @data[:architecture]  = value_at(lines[5])
      @data
    end
  end

  class ArchPackagesFile < WebFile
    def count
      @data[:packages].size
    end

    def current_package
      @data[:packages][@cursor]
    end

    def each(&block)
      @data[:packages].each(&block)
    end

    def select(&block)
      @data[:packages].select(&block)
    end

    def first
      @cursor = 0
      current_package
    end

    def next
      @cursor += 1
      current_package
    end

    def previous
      @cursor -= 1
      current_package
    end

    def last
      @cursor = count
      current_package
    end

    def parse
      lines = File.read(@copy)
      pkgs  = lines.split("\n\n")
      @data[:packages] = []
      pkgs.each do |pkg|
        x = {}
        Devuan::Packages::Fields.each do |field|
          key = field.downcase.gsub('-', '_').intern
          if Devuan::Packages.stupid_field?(field)
            x[:stupid] ||= []
            x[:stupid] << field
          end
          if Devuan::Packages.multiline_field?(field)
            x[key] = value_for(field, pkg, :multiline => true)
          else
            x[key] = value_for(field, pkg)
          end
        end
        @data[:packages] << package_ds(x)
      end
      reset_cursor!
      @data
    end

    protected

    def reset_cursor!
      @cursor = 0
    end

    def value_for(field, string, options = {})
      if options[:multiline] # cut to next Field:
        $1 if string =~ /^#{field}: (.*?)(\n[A-Z][\w-]+:)/m
      else
        $1 if string =~ /^#{field}: (.*)$/
      end
    end

    protected

    # Turn raw data from Packages into a proper data structure
    def package_ds(raw)
      pkg = {
        name:           raw[:package],
        provides:       raw[:provides],
        description:    raw[:description],
        maintainer:     raw[:maintainer],
        essential:      raw[:essential],
        origin:         raw[:origin],
        priority:       raw[:priority],
        section:        raw[:section],
        task:           raw[:task],
        popcon:         popcon_for(raw[:package]),
        download_size:  raw[:size],
        installed_size: raw[:installed_size],
        architectures:  raw[:architectures],
        multi_arch:     raw[:multi_arch],
        sums: {
          md5:          raw[:md5sum],
          sha1:         raw[:sha1],
          sha256:       raw[:sha256]
        },
        links: {
          avatar:       avatar_for(raw[:package]),
          homepage:     raw[:homepage],
          issues:       "https://git.devuan.org/devuan-packages/#{raw[:package]}/issues",
          milestones:   "https://git.devuan.org/devuan-packages/#{raw[:package]}/milestones"
        },
        upstream:       upstream_info_for(raw[:package]),
        dependencies: {
          breaks:       raw[:breaks],
          conflicts:    raw[:conflicts],
          depends:      raw[:depends],
          pre_depends:  raw[:pre_depends],
          recommends:   raw[:recommends],
          suggests:     raw[:suggests]
        },
        releases: {
          stable: {
            filename:   "https://packages.devuan.org/#{raw[:filename]}",
            tarball:    nil,
            version:    raw[:version]
          },
          testing: {
            filename:   "https://packages.devuan.org/#{raw[:filename]}",
            tarball:    nil,
            version:    raw[:version]
          },
          unstable: {
            filename:   "https://packages.devuan.org/#{raw[:filename]}",
            tarball:    nil,
            version:    raw[:version]
          }
        },
        meta: {
          broken_home:   raw[:homepage].nil?,
          stupid:        !raw[:stupid].empty?,
          stupid_fields: raw[:stupid]
        }
      }
    end

    # Get avatar from gitlab
    def avatar_for(pkg)
      nil
    end

    # Get popcon info for package
    def popcon_for(pkg)
      nil
    end

    # Get upstream info for package
    def upstream_info_for(pkg)
      upstream = {
        homepage:     nil,
        issues:       nil,
        maintainer:   nil,
        version:      nil
      }
    end
  end

  module Packages

    # grep -e '^[^ :]*: ' tmp/main-binary-amd64-Packages | cut -d: -f1 | sort | uniq
    Fields = %w(Apport Architecture Breaks Bugs Build-Essential
    Built-Using Conflicts Depends Description Description-md5 Enhances
    Essential Filename Ghc-Package Gstreamer-Decoders Gstreamer-Elements
    Gstreamer-Encoders Gstreamer-Uri-Sinks Gstreamer-Uri-Sources
    Gstreamer-Version Homepage Installed-Size Lua-Versions Maintainer
    MD5sum Modaliases Multi-Arch Npp-Applications Npp-Description
    Npp-File Npp-Mimetype Npp-Name Origin Original-Maintainer
    Original-Source-Maintainer Package Package-Type Postgresql-Version
    Pre-Depends Priority Provides Python-Version Python-Versions
    Recommends Replaces Ruby-Versions Section SHA1 SHA256 Size Source
    Suggests Tag Task Version Xul-Appid)

    # Multiline fields
    MFields = %w(Description)

    # Stupid fields
    SFields = %w(Ghc-Package Gstreamer-Decoders Gstreamer-Elements
    Gstreamer-Encoders Gstreamer-Uri-Sinks Gstreamer-Uri-Sources
    Gstreamer-Version Lua-Versions Npp-Applications Npp-Description
    Npp-File Npp-Mimetype Npp-Name Postgresql-Version Python-Version
    Python-Versions Ruby-Versions Xul-Appid)

    class << self

      def multiline_field?(field)
        MFields.include?(field)
      end

      def stupid_field?(field)
        SFields.include?(field)
      end

      def release
        return @release unless @release.nil?

        @release  = Devuan::GlobalReleaseFile.new
        @release.fetch
        @release.parse
        @release
      end

      def packages
        return @packages unless @packages.nil?

        @packages = Devuan::ArchPackagesFile.new(uri: "main/binary-amd64/Packages.xz", fetch: false)
        @packages.fetch
        @packages.parse
        @packages
      end

      # TODO: ask amprolla instead
      def blacklisted
        [ 'systemd', 'systemd-sysvinit' ]
      end

      def named(name)
        packages.select { |x| x[:package] =~ /#{name}/ }
      end

      def depending_on(package)
        pattern = Regexp.new(package)
        packages.select do |x|
          x[:pre_depends] =~ pattern || x[:depends] =~ pattern
        end
      end

      def from_devuan
        packages.select { |x| x[:releases][:stable][:version] =~ /[+]devuan/ }
      end

      def save_to_yml
        File.open(BASE + '/tmp/packages.yml', 'w+') do |f|
          f.puts(YAML.dump(packages))
        end
        File.open(BASE + '/tmp/devuan_packages.yml', 'w+') do |f|
          f.puts(YAML.dump(from_devuan))
        end
        File.open(BASE + '/tmp/suspicious_packages.yml', 'w+') do |f|
          f.puts(YAML.dump(depending_on('libsystemd0')))
        end
      end

      protected

      def method_missing(*args)
        if packages.respond_to?(args.first)
          packages.send(args.shift, *args)
        else
          super
        end
      end
    end # class << self
  end # Module Packages
end # Module Devuan


release = Devuan::GlobalReleaseFile.new(fetch: false)
release.fetch
release.parse

puts "Origin: #{release.origin}, version #{release.version}"
puts "Architectures: #{release.architectures}"
puts "Retrieving package information... Might take a while..."
puts "Retrieved information about #{Devuan::Packages.count} packages."
puts "Among those, #{Devuan::Packages.from_devuan.count} packages were modified by Devuan."

puts "E.g., #{Devuan::Packages.from_devuan.first[:name]}..."

puts "Saving to YAML format"
Devuan::Packages.save_to_yml

puts "Here comes the list of devuan-modified packages..."
puts Devuan::Packages.from_devuan.map { |x| x[:name] }.join(', ')

puts "Here is a list of packages starting with devuan:"
p Devuan::Packages.named('^devuan')
    #.map { |x| x[:name] }.join(', ')

puts "#{Devuan::Packages.named('systemd').count} packages contain `systemd` in their name."

# Suspicious packages
libsystemd0_dependent = Devuan::Packages.depending_on('libsystemd0')
systemd_dependent     = Devuan::Packages.depending_on('systemd')
remaining_packages_to_fix = libsystemd0_dependent - systemd_dependent
count = remaining_packages_to_fix.count

puts "There are #{libsystemd0_dependent.count} packages depending on libsystemd0:"
p libsystemd0_dependent.class
puts libsystemd0_dependent.map { |e| e[:name] }.join(', ')

p libsystemd0_dependent.first

if count > 0
  puts "There are #{count} packages with dependency on systemd."
  puts remaining_packages_to_fix.map(&:name).join(', ') unless count > 42
else
  puts "Devuan #{release.version} is free from systemd dependencies!"
end