kmx git

require 'time'
load "#{File.dirname(__FILE__)}/rbpkg/log.rb"
load "#{File.dirname(__FILE__)}/rbpkg/repos.rb"

if File.exist?("#{File.dirname(File.dirname(__FILE__))}/etc/rbpkg.rb")
  load "#{File.dirname(File.dirname(__FILE__))}/etc/rbpkg.rb"
end

module Rbpkg
  def self.build(repos)
    repos.each do |name|
      Repos.repo(name).build()
    end
  end

  def self.cc
    if system("which cc >/dev/null 2>&1")
      "cc"
    elsif system("which gcc >/dev/null 2>&1")
      "gcc"
    elsif system("which egcc >/dev/null 2>&1")
      "egcc"
    end
  end

  def self.cc!
    r = cc()
    raise "C compiler not found" unless r
    r
  end

  def self.checkout(branch, repos)
    repos.each do |name|
      Repos.repo(name).checkout(branch)
    end
  end

  def self.ci_dir(parent = dir)
    "#{parent}/ci"
  end

  def self.ci_build_dir
    "#{ci_dir}/build/#{os}/#{target}"
  end

  def self.ci_init
    [ci_dir,
     ci_build_dir,
     ci_log_dir,
     ci_status_dir
    ].each do |d|
      FileUtils.mkdir_p(d)
    end
  end

  def self.ci_log_dir
    "#{ci_dir}/log/#{os}/#{target}/#{date}_#{$$}"
  end

  def self.ci_mux
    verbose(3, "Rbpkg.ci_mux")
    status = ci_mux_status_hosts
    final_status = ci_mux_status(status)
    log = ci_mux_log_hosts(status, final_status)
    ci_mux_log(log, final_status)
  end

  def self.ci_mux_log(log, final_status)
    Dir.chdir("#{Rbpkg.ci_dir}/log") do
      log.each do |name, sublogs|
        puts "Rbpkg.ci_mux_log name=#{name.inspect} sublogs=#{sublogs.inspect}"
        File.open("#{name}.html", "w") do |output|
          puts({name: name, final_status: final_status[name]}.inspect)
          s = final_status[name]
          img = status_img(s)
          output.puts <<-EOF
<!DOCTYPE html>
<html lang="en">
  <head>
    <meta charset="utf-8"/>
    <meta http-equiv="X-UA-Compatible" content="IE=edge"/>
    <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
    <title>#{name.to_html}</title>
    <link rel="stylesheet" href="/_assets/app.css">
    <script defer type="text/javascript" src="/_assets/app.js"></script>
    <link rel="icon" type="image/png" sizes="64x64" href="${IMG}">
  </head>
  <body>
    <div class="ci-header">
      <a href="./"><i class="fas fa-asterisk"></i> All logs</a>
    </div>
    <h1>
      <img src="#{img}" class="status-#{s.to_s.to_html}"/>
      #{name.to_html}
    </h1>
    <ul class="list-group">
EOF
          sublogs.each do |d, sublog|
            sublog_img = status_img(sublog[:status])
            sublog_html = (sublog[:html] || sublog[:log]).to_html
            sublog_status = sublog[:status].to_s.to_html
            output.puts <<-EOF
      <li class="list-group-item">
        <img src="#{sublog_img}" class="status-#{sublog_status}"/>
        <a href="#{sublog_html}">#{sublog_html}</a>
      </li>
EOF
          end
          output.puts <<-EOF
    </ul>
  </body>
</html>
EOF
          output.close
        end
      end
    end
  end

  def self.ci_mux_log_hosts(status, final_status)
    log = {}
    Dir.chdir("#{Rbpkg.ci_dir}/log") do
      Dir["*/*"].each do |d|
        puts "ci_mux_log_hosts d=#{d.inspect}"
        if File.directory?(d)
          Dir.chdir(d) do
            Dir["*/*.log"].each do |subdir_file|
              name = File.basename(subdir_file, ".log")
              d_name = "#{d}/#{name}"
              file = "#{name}.log"
              if ! File.exist?(file) || File.mtime(file) <= File.mtime(subdir_file)
                FileUtils.cp(subdir_file, file)
                FileUtils.touch(file, mtime: File.mtime(subdir_file))
                subdir_file_html = "#{subdir_file}.html"
                if File.exist?(subdir_file_html)
                  file_html = "#{file}.html"
                  FileUtils.cp(subdir_file_html, file_html)
                  FileUtils.touch(file_html, mtime: File.mtime(subdir_file_html))
                  html = "#{d}/#{subdir_file_html}"
                else
                  html = nil
                end
                log_item = {log: "#{d}/#{subdir_file}",
                            html: html,
                            status: status[name][d_name]}
                log[name] = {} unless log[name]
                log[name][d_name] = log_item
              end
            end
          end
        end
      end
    end
    log
  end

  def self.ci_mux_status(status)
    final_status = {}
    Dir.chdir("#{ci_dir}/status") do
      status.each do |name, s|
        puts "#{name} -> #{s}"
        s.each do |subdir, subdir_status|
          case subdir_status
          when :running
            final_status[name] = :running
          when :ko
            if final_status[name] != :running
              final_status[name] = :ko
            end
          when :ok
            if ! final_status[name]
              final_status[name] = :ok
            end
          end
        end
        File.write("#{name}.status", (final_status[name] || :unknown).to_s + "\n")
      end
    end
    final_status
  end

  def self.ci_mux_status_hosts
    status = {}
    Dir.chdir("#{ci_dir}/status") do
      Dir["*/*"].each do |d|
        Dir.chdir(d) do
          Dir["*/*.status"].each do |subdir_file|
            s = File.read(subdir_file).strip.to_sym
            raise "invalid status" unless [:running, :ko, :ok].include?(s)
            name = File.basename(subdir_file, ".status")
            d_name = "#{d}/#{name}"
            file = "#{name}.status"
            if ! File.exist?(file) || File.mtime(file) <= File.mtime(subdir_file)
              c = "cp -a #{sh_quote(subdir_file)} #{sh_quote(file)}"
              raise "command failed: #{c}" unless system(c)
              if ! status[name]
                status[name] = {}
              end
              status[name][d_name] = s
            end
          end
        end
      end
    end
    status
  end

  def self.ci_remote(repo, branch, commit, host)
    verbose(3, "Rbpkg.ci_remote(#{repo.inspect}, #{branch.inspect}, #{commit.inspect}, #{host.inspect})")
    sh_host = sh_quote(host)
    cmd "ssh #{sh_host} rbpkg/bin/rbpkg_ci #{sh_quote(repo)} #{sh_quote(branch)} #{sh_quote(commit)}"
    sh_ci_dir = sh_quote(ci_dir)
    if `ssh #{sh_host} uname`.strip == "Darwin"
      sh_remote_ci_dir = sh_ci_dir.gsub(/^\/home\//, "/Users/")
    else
      sh_remote_ci_dir = sh_ci_dir
    end
    cmd! "rsync -a --exclude build #{sh_host}:#{sh_ci_dir}/. #{sh_ci_dir}"
  end

  def self.ci_status_dir
    "#{ci_dir}/status/#{os}/#{target}/#{date}_#{$$}"
  end

  def self.clean_all(repos)
    repos.each do |name|
      Repos.repo(name).clean_all()
    end
  end

  def self.clean_build(repos)
    repos.each do |name|
      Repos.repo(name).clean_build()
    end
  end

  def self.clean_configure(repos)
    repos.each do |name|
      Repos.repo(name).clean_configure()
    end
  end

  def self.clean_fake(repos)
    repos.each do |name|
      Repos.repo(name).clean_fake()
    end
  end

  def self.clean_package(repos)
    repos.each do |name|
      Repos.repo(name).clean_package()
    end
  end

  def self.clean_sources(repos)
    repos.each do |name|
      Repos.repo(name).clean_sources()
    end
  end

  def self.configure(repos)
    repos.each do |name|
      Repos.repo(name).configure()
    end
  end

  def self.date
    $date || $date = Time.now.utc.iso8601
  end

  @@dir = nil
  def self.dir
    @@dir || @@dir = (ENV["RBPKG_DIR"] || "#{ENV["HOME"]}/rbpkg")
  end

  def self.ensure_dir(path)
    if ! File.directory?(path)
      cmd! "mkdir -p #{sh_quote(path)}"
    end
  end

  def self.fake(repos)
    repos.each do |name|
      Repos.repo(name).fake()
    end
  end

  def self.fake_dir
    "#{target_dir}/fake"
  end

  def self.fetch(repos)
    repos.each do |name|
      Repos.repo(name).fetch()
    end
  end

  def self.git_clone(repos)
    repos.each do |name|
      Repos.repo(name).git_clone()
    end
  end

  def self.info(repos)
    global_info = inspect()
    $log.puts global_info
    puts global_info
    repos.each do |name|
      repo_info = Repos.repo(name).inspect()
      $log.puts repo_info
      puts repo_info
    end
  end

  def self.init(name, verbose_level, log_dir, status_dir)
    @@verbose_level = verbose_level
    [dir,
     lock_dir,
     log_dir
    ].each do |d|
      FileUtils.mkdir_p(d)
    end
    $log = Rbpkg::Log
    Rbpkg::Log.init(log_dir, name, status_dir)
    Rbpkg.lock(name)
  end

  def self.inspect
    """%Rbpkg{
 CC: #{cc.inspect},
 dir: #{dir.inspect},
 ncpu: #{ncpu.inspect},
 target: #{target.inspect}
}"""
  end

  def self.install(repos)
    repos.each do |name|
      Repos.repo(name).install()
    end
  end

  def self.installed_dir
    "#{dir}/var/db/installed"
  end

  def self.lib_dir
    "#{dir}/lib"
  end

  def self.lock(name)
    path = lock_path(name)
    if File.file?(path)
      verbose(2, "Waiting for lock #{path}")
      while File.file?(path)
        sleep 1
      end
    end
    File.write(path, "")
    at_exit { FileUtils.rm(path) }
  end

  def self.lock_dir
    "#{dir}/var/lock"
  end

  def self.lock_path(name)
    "#{lock_dir}/#{name}.lock"
  end

  def self.log_dir
    "#{dir}/var/log/rbpkg/#{date}_#{$$}"
  end

  @@ncpu = nil
  def self.ncpu
    @@ncpu || @@ncpu = self.ncpu_fetch
  end

  def self.ncpu_fetch
    `which nproc >/dev/null 2>&1 && nproc || { which sysctl >/dev/null 2>&1 && sysctl -n hw.ncpu; } || echo 1`.strip.to_i
  end

  def self.obj_dir
    "#{target_dir}/obj"
  end

  @@os = nil
  def self.os
    os_fetch unless @@os
    @@os
  end

  @@os_dir = nil
  def self.os_dir
    os_fetch unless @@os_dir
    @@os_dir
  end

  def self.os_fetch
    uname = `uname`.strip
    raise "uname failed" unless $?.success?
    if uname == "Linux"
      if File.exist?("/etc/fedora-release")
        distrib = `cat /etc/fedora-release | cut -d ' ' -f 1`.strip
      else
        distrib = `uname -a | grep -io -e Debian -e gentoo -e Ubuntu | sed -e 's|gentoo|Gentoo|'`.strip
      end
      @@os_dir = "#{uname}/#{distrib}"
      @@os = "#{uname}_#{distrib}"
    else
      @@os = uname
      @@os_dir = uname
    end
  end

  def self.package(repos)
    repos.each do |name|
      Repos.repo(name).package()
    end
  end

  def self.package_dir
    "#{target_dir}/package"
  end

  def self.prefix
    dir
  end

  def self.pull(repos)
    repos.each do |name|
      Repos.repo(name).pull()
    end
  end

  def self.src_dir
    "#{dir}/src"
  end

  def self.status_img(status)
    path = "#{Rbpkg.dir}/share/icon/64x64/status_#{status}.png"
    file = File.read(path)
    base64 = Base64.encode64(file)
    "data:image/png;base64,#{base64}"
  end

  def self.tag_dir
    "#{target_dir}/tag"
  end

  @@target = nil
  def self.target
    @@target || @@target = `LC_ALL=C.UTF-8 #{sh_quote(cc!)} -v 2>&1 | grep '^Target: ' | head -n 1 | cut -c 9-`.strip
  end

  def self.target_dir
    "#{dir}/target/#{target}"
  end

  def self.test(repos)
    repos.each do |name|
      Repos.repo(name).test()
    end
  end

  def self.uninstall(repos)
    repos.each do |name|
      Repos.repo(name).uninstall()
    end
  end

  def self.upgrade(repos)
    upgrade_self
    repos.each do |name|
      Repos.repo(name).upgrade()
    end
  end

  def self.upgrade_self
    verbose 3, "Rbpkg.upgrade_self"
    cmd! "cd #{sh_quote(dir)} && git pull"
    load "#{dir}/lib/rbpkg.rb"
  end

  def self.verbose_level
    @@verbose_level
  end
end

def cmd(string)
  verbose(1, string)
  IO.popen("#{string} 2>&1", "r") do |pipe|
    while line = pipe.gets()
      $log.puts line
      puts line if Rbpkg.verbose_level > 1
      $stdout.flush
    end
    pipe.close
    $?.success?
  end
end

def cmd!(string)
  r = cmd(string)
  raise "command failed: #{string}" unless r
  true
end

def sh_quote(str)
  if str.scan("\n") == [] && str.match?(/^[-+\/=.,:^_0-9A-Za-z]*$/)
    str
  else
    "\"" + str.gsub(/([$`\\\"])/, "\\\1") + "\""
  end
end
      
def verbose(level, msg)
  colored = case level
            when -1
              "\33[0;31m#{msg}\33[0m\n"
            when 0
              msg
            when 1
              "\33[0;34m#{msg}\33[0m\n"
            when 2
              "\33[0;35m#{msg}\33[0m\n"
            when 3
              "\33[0;35m#{msg}\33[0m\n"
            else
              raise "unknown verbose level: #{level.inspect}"
            end
  $log.puts colored
  STDOUT.puts colored if level <= Rbpkg.verbose_level
end
kmx.io/rbpkg/lib/rbpkg.rb

Commit

lib/rbpkg.rb
