Skip to content

Commit e9b104d

Browse files
committed
dev: scripts to package gems and assert on their contents
see CONTRIBUTING.md for an explanation
1 parent 2965ef9 commit e9b104d

7 files changed

Lines changed: 332 additions & 5 deletions

File tree

CONTRIBUTING.md

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Contributing to sqlite3-ruby
2+
3+
**This document is a work-in-progress.**
4+
5+
This doc is a short introduction on how to modify and maintain the sqlite3-ruby gem.
6+
7+
8+
## Building gems
9+
10+
As a prerequisite please make sure you have `docker` correctly installed, so that you're able to cross-compile the native gems.
11+
12+
Run `bin/build-gems` which will package gems for all supported platforms, and run some basic sanity tests on those packages using `bin/test-gem-set` and `bin/test-gem-file-contents`.
13+
14+
15+
## Making a release
16+
17+
A quick checklist:
18+
19+
- [ ] make sure CI is green!
20+
- [ ] update `CHANGELOG.md` and `lib/sqlite3/version.rb` including `VersionProxy::{MINOR,TINY}`
21+
- [ ] create a git tag using a format that matches the pattern `v\d+\.\d+\.\d+`, e.g. `v1.3.13`
22+
- [ ] run `bin/build-gems` and make sure it completes and all the tests pass
23+
- [ ] `for g in gems/*.gem ; do gem push $g ; done`
24+
- [ ] create a release at https://github.com/sparklemotion/sqlite3-ruby/releases and include sha2 checksums

bin/build-gems

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#! /usr/bin/env bash
2+
#
3+
# script to build gems for all relevant platforms
4+
#
5+
set -o errexit
6+
set -o nounset
7+
set -x
8+
9+
rm -rf tmp pkg gems
10+
mkdir -p gems
11+
12+
# prelude: vendor dependencies
13+
bundle update
14+
bundle package
15+
16+
# safety check: let's check that things work
17+
bundle exec rake clean clobber
18+
bundle exec rake compile test
19+
20+
# package the gems, including precompiled native
21+
bundle exec rake clean clobber
22+
bundle exec rake gem:all
23+
cp -v pkg/sqlite3*.gem gems
24+
25+
# test those gem files!
26+
bin/test-gem-set gems/*.gem
27+
28+
# checksums should be included in the release notes
29+
pushd gems
30+
ls *.gem | sort | xargs sha256sum
31+
popd

bin/test-gem-build

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,8 @@ else
2626
bundle exec rake gem:${BUILD_NATIVE_GEM}:buildit
2727
fi
2828

29+
./bin/test-gem-file-contents pkg/*.gem
30+
2931
mkdir -p ${OUTPUT_DIR}
3032
cp -v pkg/*.gem ${OUTPUT_DIR}
3133
ls -l ${OUTPUT_DIR}/*
32-
33-
# make sure github actions cache can read the file
34-
chmod 666 ports/archives/*

bin/test-gem-file-contents

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
#! /usr/bin/env ruby
2+
#
3+
# this script is intended to run as part of the CI test suite.
4+
#
5+
# it inspects the contents of a gem file -- both the files and the gemspec -- to ensure we're
6+
# packaging what we expect, and that we're not packaging anything we don't expect.
7+
#
8+
# this file isn't in the `test/` subdirectory because it's intended to be run standalone against a
9+
# built gem file (and not against the source code or behavior of the gem itself).
10+
#
11+
require "bundler/inline"
12+
13+
gemfile do
14+
source "https://rubygems.org"
15+
gem "minitest"
16+
gem "minitest-reporters"
17+
end
18+
19+
require "yaml"
20+
21+
def usage_and_exit(message = nil)
22+
puts "ERROR: #{message}" if message
23+
puts "USAGE: #{File.basename(__FILE__)} <gemfile> [options]"
24+
exit(1)
25+
end
26+
27+
usage_and_exit if ARGV.include?("-h")
28+
usage_and_exit unless (gemfile = ARGV[0])
29+
usage_and_exit("#{gemfile} does not exist") unless File.file?(gemfile)
30+
usage_and_exit("#{gemfile} is not a gem") unless /\.gem$/.match?(gemfile)
31+
gemfile = File.expand_path(gemfile)
32+
33+
gemfile_contents = Dir.mktmpdir do |dir|
34+
Dir.chdir(dir) do
35+
unless system("tar -xf #{gemfile} data.tar.gz")
36+
raise "could not unpack gem #{gemfile}"
37+
end
38+
39+
%x(tar -ztf data.tar.gz).split("\n")
40+
end
41+
end
42+
43+
gemspec = Dir.mktmpdir do |dir|
44+
Dir.chdir(dir) do
45+
unless system("tar -xf #{gemfile} metadata.gz")
46+
raise "could not unpack gem #{gemfile}"
47+
end
48+
49+
YAML.unsafe_load(%x(gunzip -c metadata.gz))
50+
end
51+
end
52+
53+
if ARGV.include?("-v")
54+
puts "---------- gemfile contents ----------"
55+
puts gemfile_contents
56+
puts
57+
puts "---------- gemspec ----------"
58+
puts gemspec.to_ruby
59+
puts
60+
end
61+
62+
require "minitest/autorun"
63+
require "minitest/reporters"
64+
Minitest::Reporters.use!([Minitest::Reporters::SpecReporter.new])
65+
66+
puts "Testing '#{gemfile}' (#{gemspec.platform})"
67+
describe File.basename(gemfile) do
68+
let(:all_supported_ruby_versions) { ["1.9.2", "2.0", "2.1", "2.2", "2.3", "2.4", "2.5", "2.6", "2.7", "3.0", "3.1"] }
69+
let(:native_supported_ruby_versions) { ["2.6", "2.7", "3.0", "3.1"] }
70+
let(:ucrt_supported_ruby_versions) { ["3.1"] }
71+
let(:platform_supported_ruby_versions) do
72+
if gemspec.platform.to_s == "x64-mingw-ucrt"
73+
ucrt_supported_ruby_versions
74+
elsif gemspec.platform.to_s == "x64-mingw32"
75+
native_supported_ruby_versions - ucrt_supported_ruby_versions
76+
elsif gemspec.platform.cpu
77+
native_supported_ruby_versions
78+
else
79+
all_supported_ruby_versions
80+
end
81+
end
82+
83+
describe "setup" do
84+
it "gemfile contains some files" do
85+
actual = gemfile_contents.length
86+
assert_operator(actual, :>, 10, "expected gemfile to contain more than #{actual} files")
87+
end
88+
89+
it "gemspec is a Gem::Specification" do
90+
assert_equal(Gem::Specification, gemspec.class)
91+
end
92+
end
93+
94+
describe "all platforms" do
95+
["lib", "test"].each do |dir|
96+
it "contains every ruby file in #{dir}/" do
97+
expected = %x(git ls-files #{dir}).split("\n").grep(/\.rb$/).sort
98+
skip "looks like this isn't a git repository" if expected.empty?
99+
actual = gemfile_contents.select { |f| f.start_with?("#{dir}/") }.grep(/\.rb$/).sort
100+
assert_equal(expected, actual)
101+
end
102+
end
103+
end
104+
105+
describe "ruby platform" do
106+
it "depends on mini_portile2" do
107+
assert(gemspec.dependencies.find { |d| d.name == "mini_portile2" })
108+
end
109+
110+
it "contains extension C and header files" do
111+
assert_equal(6, gemfile_contents.count { |f| File.fnmatch?("ext/**/*.c", f) })
112+
assert_equal(6, gemfile_contents.count { |f| File.fnmatch?("ext/**/*.h", f) })
113+
end
114+
115+
it "includes C files in extra_rdoc_files" do
116+
assert_equal(6, gemspec.extra_rdoc_files.count { |f| File.fnmatch?("ext/**/*.c", f) })
117+
end
118+
119+
it "contains the port files" do
120+
actual_ports = gemfile_contents.grep(%r{^ports/})
121+
assert_equal(1, actual_ports.count { |f| File.fnmatch?("ports/archives/sqlite-autoconf-*.tar.gz", f) })
122+
assert_equal(1, actual_ports.length)
123+
end
124+
125+
it "contains the patch files" do
126+
assert_operator(gemfile_contents.grep(%r{^patches/}).length, :>, 0)
127+
end
128+
129+
it "sets metadata for msys2" do
130+
refute_nil(gemspec.metadata["msys2_mingw_dependencies"])
131+
end
132+
133+
it "sets required_ruby_version appropriately" do
134+
all_supported_ruby_versions.each do |v|
135+
assert(
136+
gemspec.required_ruby_version.satisfied_by?(Gem::Version.new(v)),
137+
"required_ruby_version='#{gemspec.required_ruby_version}' should support ruby #{v}",
138+
)
139+
end
140+
end
141+
end if gemspec.platform == Gem::Platform::RUBY
142+
143+
describe "native platform" do
144+
it "does not depend on mini_portile2" do
145+
refute(gemspec.dependencies.find { |d| d.name == "mini_portile2" })
146+
end
147+
148+
it "contains extension C and header files" do
149+
assert_equal(6, gemfile_contents.count { |f| File.fnmatch?("ext/**/*.c", f) })
150+
assert_equal(6, gemfile_contents.count { |f| File.fnmatch?("ext/**/*.h", f) })
151+
end
152+
153+
it "includes C files in extra_rdoc_files" do
154+
assert_equal(6, gemspec.extra_rdoc_files.count { |f| File.fnmatch?("ext/**/*.c", f) })
155+
end
156+
157+
it "does not contain the port files" do
158+
assert_empty(gemfile_contents.grep(%r{^ports/}))
159+
end
160+
161+
it "does not contain the patch files" do
162+
assert_empty(gemfile_contents.grep(%r{^patches/}))
163+
end
164+
165+
it "contains expected shared library files " do
166+
platform_supported_ruby_versions.each do |version|
167+
actual = gemfile_contents.find do |p|
168+
File.fnmatch?("lib/sqlite3/#{version}/sqlite3_native.{so,bundle}", p, File::FNM_EXTGLOB)
169+
end
170+
assert(actual, "expected to find shared library file for ruby #{version}")
171+
end
172+
173+
actual = gemfile_contents.find do |p|
174+
File.fnmatch?("lib/sqlite3/sqlite3_native.{so,bundle}", p, File::FNM_EXTGLOB)
175+
end
176+
refute(actual, "did not expect to find shared library file in lib/sqlite3")
177+
178+
actual = gemfile_contents.find_all do |p|
179+
File.fnmatch?("lib/sqlite3/**/*.{so,bundle}", p, File::FNM_EXTGLOB)
180+
end
181+
assert_equal(
182+
platform_supported_ruby_versions.length,
183+
actual.length,
184+
"did not expect extra shared library files",
185+
)
186+
end
187+
188+
it "sets required_ruby_version appropriately" do
189+
unsupported_versions = all_supported_ruby_versions - platform_supported_ruby_versions
190+
platform_supported_ruby_versions.each do |v|
191+
assert(
192+
gemspec.required_ruby_version.satisfied_by?(Gem::Version.new(v)),
193+
"required_ruby_version='#{gemspec.required_ruby_version}' should support ruby #{v}",
194+
)
195+
end
196+
unsupported_versions.each do |v|
197+
refute(
198+
gemspec.required_ruby_version.satisfied_by?(Gem::Version.new(v)),
199+
"required_ruby_version='#{gemspec.required_ruby_version}' should not support ruby #{v}",
200+
)
201+
end
202+
end
203+
204+
it "does not set metadata for msys2" do
205+
assert_nil(gemspec.metadata["msys2_mingw_dependencies"])
206+
end
207+
end if gemspec.platform.is_a?(Gem::Platform) && gemspec.platform.cpu
208+
end

bin/test-gem-set

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#! /usr/bin/env bash
2+
#
3+
# script to test a set of gem files
4+
#
5+
set -o errexit
6+
set -o nounset
7+
set -o pipefail
8+
9+
gems=$*
10+
gem_platform_local=`ruby -e "puts Gem::Platform.local.to_s"`
11+
12+
function remove_all_sqlite3s {
13+
yes | gem uninstall --force sqlite3 || true
14+
}
15+
16+
function test_installation {
17+
gem=$1
18+
remove_all_sqlite3s
19+
gem install --local $gem
20+
ruby -r sqlite3 -e 'pp SQLite3::SQLITE_VERSION, SQLite3::SQLITE_LOADED_VERSION'
21+
22+
if [[ $gem =~ sqlite3-[^-]*\.gem ]] ; then
23+
remove_all_sqlite3s
24+
gem install --local $gem -- --enable-system-libraries
25+
ruby -r sqlite3 -e 'pp SQLite3::SQLITE_VERSION, SQLite3::SQLITE_LOADED_VERSION'
26+
fi
27+
}
28+
29+
for gem in $gems ; do
30+
./bin/test-gem-file-contents $gem
31+
done
32+
33+
for gem in $gems ; do
34+
if [[ $gem =~ sqlite3-[^-]+(-${gem_platform_local})?\.gem$ ]] ; then
35+
test_installation $gem
36+
fi
37+
done

rakelib/native.rake

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,33 @@ ENV["RUBY_CC_VERSION"] = cross_rubies.join(":")
2121
Gem::PackageTask.new(SQLITE3_SPEC).define # packaged_tarball version of the gem for platform=ruby
2222
task "package" => cross_platforms.map { |p| "gem:#{p}" } # "package" task for all the native platforms
2323

24+
def gem_build_path
25+
File.join("pkg", SQLITE3_SPEC.full_name)
26+
end
27+
28+
def add_file_to_gem(relative_source_path)
29+
if relative_source_path.nil? || !File.exist?(relative_source_path)
30+
raise "Cannot find file '#{relative_source_path}'"
31+
end
32+
33+
dest_path = File.join(gem_build_path, relative_source_path)
34+
dest_dir = File.dirname(dest_path)
35+
36+
mkdir_p(dest_dir) unless Dir.exist?(dest_dir)
37+
rm_f(dest_path) if File.exist?(dest_path)
38+
safe_ln(relative_source_path, dest_path)
39+
40+
SQLITE3_SPEC.files << relative_source_path
41+
end
42+
43+
task gem_build_path do
44+
archive = Dir.glob(File.join("ports", "archives", "sqlite-autoconf-*.tar.gz")).first
45+
add_file_to_gem(archive)
46+
47+
patches = %x(#{["git", "ls-files", "patches"].shelljoin}).split("\n").grep(/\.patch\z/)
48+
patches.each { |patch| add_file_to_gem patch }
49+
end
50+
2451
Rake::ExtensionTask.new("sqlite3_native", SQLITE3_SPEC) do |ext|
2552
ext.ext_dir = "ext/sqlite3"
2653
ext.lib_dir = "lib/sqlite3"
@@ -29,7 +56,7 @@ Rake::ExtensionTask.new("sqlite3_native", SQLITE3_SPEC) do |ext|
2956
ext.cross_config_options << "--enable-cross-build" # so extconf.rb knows we're cross-compiling
3057
ext.cross_compiling do |spec|
3158
# remove things not needed for precompiled gems
32-
spec.files.reject! { |file| File.fnmatch?("*.tar.gz", file) } # TODO check
59+
spec.dependencies.reject! { |dep| dep.name == "mini_portile2" }
3360
spec.metadata.delete('msys2_mingw_dependencies')
3461
end
3562
end
@@ -56,7 +83,7 @@ namespace "gem" do
5683
end
5784

5885
desc "build native gem for all platforms"
59-
multitask "all" => [cross_platforms, "gem"].flatten
86+
task "all" => [cross_platforms, "gem"].flatten
6087
end
6188

6289
desc "Temporarily set VERSION to a unique timestamp"

sqlite3.gemspec

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ Gem::Specification.new do |s|
3838
".gemtest",
3939
"API_CHANGES.md",
4040
"CHANGELOG.md",
41+
"CONTRIBUTING.md",
4142
"ChangeLog.cvs",
4243
"Gemfile",
4344
"LICENSE",

0 commit comments

Comments
 (0)