File Coverage

File:lib/CheckSpelling/Apply.pm
Coverage:43.2%

linestmtbrancondsubtimecode
1package CheckSpelling::Apply;
2sub tear_here {
3
9
9
17
  my ($exit) = @_;
4
9
6
  our $exited;
5
9
100
16
  return if defined $exited;
6
5
74
  print STDERR "\n<<<TEAR HERE<<<exit: $exit\n";
7
5
43
  print STDOUT "\n<<<TEAR HERE<<<exit: $exit\n";
8
5
14
  $exited = $exit;
9}
10sub die_custom {
11
5
5
12
  my ($line, $message) = @_;
12
5
4
  our $program;
13
5
26
  print STDERR "$message at $program line $line.\n";
14
5
8
  tear_here(1);
15
5
77
  die "stopping";
16}
17#!/usr/bin/env perl
18":" || q@<<"=END_OF_PERL"@;
19
20
1
1
1
1
118781
2
39
use Symbol 'gensym';
21
1
1
1
1
171
1250
28
use IPC::Open3;
22
1
1
1
1
5
1
27
use File::Basename qw(dirname);
23
1
1
1
1
2
1
17
use File::Path qw(make_path);
24
1
1
1
1
187
306
30
use File::Spec::Functions qw(catfile path);
25
1
1
1
1
3
0
19
use File::Temp qw/ tempfile tempdir /;
26
1
1
1
1
2
0
17
use JSON::PP;
27
1
1
1
1
3
0
3005
use warnings;
28
29my @safe_path = qw(
30    /opt/homebrew/bin
31    /opt/homebrew/sbin
32    /usr/local/bin
33    /usr/bin
34    /bin
35    /usr/sbin
36    /sbin
37);
38
39my $bin = glob("~/bin");
40push @safe_path, $bin if -d $bin;
41
42my $ua = 'check-spelling-agent/0.0.4';
43
44$ENV{'PATH'} = join ':', @safe_path unless defined $ENV{SYSTEMROOT};
45
46sub check_exists_command {
47
0
0
0
    my ($program) = @_;
48
49
0
0
    my @path = path;
50
0
0
    my @pathext = ('');
51
52
0
0
0
    if ($^O eq 'MSWin32') {
53
0
0
0
0
        push @pathext, map { lc } split /;/, $ENV{PATHEXT};
54    }
55
56
0
0
    for my $dir (@path) {
57
0
0
        for my $suffix (@pathext) {
58
0
0
            my $f = catfile $dir, "$program$suffix";
59
0
0
0
            return $f if -x $f;
60        }
61    }
62}
63
64sub needs_command_because {
65
0
0
0
    my ($program, $reason) = @_;
66
0
0
0
    return if check_exists_command($program);
67
0
0
    die_custom 51, 'Please install `'.$program.'` - it is needed to '.$reason;
68}
69
70sub check_basic_tools {
71
0
0
0
    needs_command_because('git', 'interact with git repositories');
72
0
0
    needs_command_because('curl', 'download other tools');
73
0
0
    needs_command_because('gh', 'interact with github');
74    #needs_command_because('magic-magic', 'debugging');
75}
76
77sub get_token {
78
4
4
3
    our $token;
79
4
100
67
21
    return $token if defined $token && $token ne '';
80
1
33
3
    $token = $ENV{'GH_TOKEN'} || $ENV{'GITHUB_TOKEN'};
81
1
50
33
6
    return $token if defined $token && $token ne '';
82
0
0
    my ($err, $exit);
83
0
0
    ($token, $err, $exit) = capture_system('gh', 'auth', 'token');
84
0
0
    chomp $token;
85
0
0
    chomp $err;
86
0
0
    return ($token, $err, $exit);
87};
88
89sub download_with_curl {
90
1
1
3
    my ($url, $dest, $flags) = @_;
91
1
50
3
    $flags = '-fsL' unless defined $flags;
92
1
78819
    system('curl',
93        '--connect-timeout', 3,
94        '-A', $ua,
95        $flags,
96        '-o', $dest,
97        $url
98    );
99}
100
101sub tempfile_name {
102
6
6
26
    my ($fh, $filename) = tempfile();
103
6
1283
    close $fh;
104
6
14
    return $filename;
105}
106
107sub strip_comments {
108
4
4
4
    my ($file) = @_;
109
4
14
    my ($fh, $filename) = tempfile();
110
4
818
    open INPUT, '<', $file;
111
4
49
    while (<INPUT>) {
112
1848
100
962
        next if /^\s*(?:#.*)/;
113
1821
1223
        print $fh $_;
114    }
115
4
12
    close INPUT;
116
4
44
    close $fh;
117
4
11
    return $filename;
118}
119
120sub capture_system {
121
17
17
57
    my @args = @_;
122
17
79
    my $pid = open3(my $child_in, my $child_out, my $child_err = gensym, @args);
123
17
46581
    my (@err, @out);
124
17
2332369
    while (my $output = <$child_out>) {
125
3
1839
        push @out, $output;
126    }
127
17
526
    while (my $error = <$child_err>) {
128
16
75
        push @err, $error;
129    }
130
17
297
    waitpid( $pid, 0 );
131
17
90
    my $child_exit_status = $?;
132
17
60
    my $output_joined = join '', @out;
133
17
54
    my $error_joined = join '', @err;
134
17
597
    return ($output_joined, $error_joined, $child_exit_status);
135}
136
137sub capture_merged_system {
138
10
10
34
    my ($output_joined, $error_joined, $child_exit_status) = capture_system(@_);
139
10
50
48
    my $joiner = ($output_joined ne '') ? "\n" : '';
140
10
51
    return ($output_joined.$joiner.$error_joined, $child_exit_status);
141}
142
143sub compare_files {
144
2
2
1595
    my ($one, $two) = @_;
145
2
6
    my $one_stripped = strip_comments($one);
146
2
1
    my $two_stripped = strip_comments($two);
147
2
1
    my $exit_code;
148
2
5
    (undef, undef, $exit_code) = capture_system(
149            'diff',
150            '-qwB',
151            $one_stripped, $two_stripped
152        );
153
2
50
5
    if ($? == -1) {
154
0
0
        print "could not compare '$one' and '$two': $!\n";
155
0
0
        return 0;
156    }
157
2
50
8
    if ($? & 127) {
158
0
0
0
        printf "child died with signal %d, %s core dump\n",
159        ($? & 127),  ($? & 128) ? 'with' : 'without';
160
0
0
        return 0;
161    }
162
2
50
3
    return 0 if $? == 0;
163
2
11
    return 1;
164}
165
166my $bash_script=q{
167=END_OF_PERL@
168# bash
169set -e
170if [ "$OUTPUT" = "$ERROR" ]; then
171    ("$@" 2>&1) > "$OUTPUT"
172else
173    "$@" > "$OUTPUT" 2> "$ERROR"
174fi
175exit
176};
177
178sub check_current_script {
179
1
50
1
121418
    if ("$0" eq '-') {
180
0
0
        my ($bash_script) = @_;
181
0
0
        my $fh;
182
0
0
        ($fh, $0) = tempfile();
183
0
0
        $bash_script =~ s/^=.*\@$//m;
184
0
0
        print $fh $bash_script;
185
0
0
        close $fh;
186
0
0
        return;
187    }
188
1
4
    my $filename = tempfile_name();
189
1
1
    my $source = 'https://raw.githubusercontent.com/check-spelling/check-spelling/prerelease/apply.pl';
190
1
2
    download_with_curl($source, $filename);
191
1
50
47
    if ($? == 0) {
192
1
50
20
        if (compare_files($filename, $0)) {
193
1
20
            print "Current apply script differs from '$source' (locally downloaded to `$filename`). You may wish to upgrade.\n";
194        }
195    }
196}
197
198sub die_with_message {
199
5
5
4
    our $program;
200
5
11
    my ($gh_err_text) = @_;
201
5
50
33
27
    if ($gh_err_text =~ /error connecting to / && $gh_err_text =~ /check your internet connection/) {
202
0
0
        print "$program: Internet access may be limited. Check your connection (this often happens with lousy cable internet service providers where their CG-NAT or whatever strands the modem).\n\n$gh_err_text";
203
0
0
0
0
        tear_here(5); die_custom 187, "exiting";
204    }
205
5
100
23
    if ($gh_err_text =~ /proxyconnect tcp:.*connect: connection refused/) {
206
1
35
        print "$program: Proxy is not accepting connections.\n";
207
1
4
        for my $proxy (qw(http_proxy HTTP_PROXY https_proxy HTTPS_PROXY)) {
208
4
100
11
            if (defined $ENV{$proxy}) {
209
1
5
                print "  $proxy: '$ENV{$proxy}'\n";
210            }
211        }
212
1
3
        print "\n$gh_err_text";
213
1
1
6
3
        tear_here(6); die_custom 197, "exiting";
214    }
215
4
100
14
    if ($gh_err_text =~ /dial unix .*: connect: .*/) {
216
1
33
        print "$program: Unix http socket is not working.\n";
217
1
30332
        my $gh_http_unix_socket = `gh config get http_unix_socket`;
218
1
31
        print "  http_unix_socket: $gh_http_unix_socket\n";
219
1
5
        print "\n$gh_err_text";
220
1
1
11
7
        tear_here(7); die_custom 204, "exiting";
221    }
222}
223
224sub gh_is_happy_internal {
225
5
5
17
    my ($output, $exit_code) = capture_merged_system(qw(gh api /installation/repositories));
226
5
50
13
    return ($exit_code, $output) if $exit_code == 0;
227
5
16
    ($output, $exit_code) = capture_merged_system(qw(gh api /user));
228
5
16
    return ($exit_code, $output);
229}
230
231sub gh_is_happy {
232
3
3
2
    my ($program) = @_;
233
3
13
    my ($gh_auth_status, $gh_status_lines) = gh_is_happy_internal();
234
3
50
11
    return 1 if $gh_auth_status == 0;
235
3
10
    die_with_message($gh_status_lines);
236
237
1
1
    my @problematic_env_variables;
238
1
2
    for my $variable (qw(GH_TOKEN GITHUB_TOKEN GITHUB_ACTIONS CI)) {
239
4
100
9
        if (defined $ENV{$variable}) {
240
2
27
            delete $ENV{$variable};
241
2
3
            push @problematic_env_variables, $variable;
242
2
4
            ($gh_auth_status, $gh_status_lines) = gh_is_happy_internal();
243
2
50
11
            if ($gh_auth_status == 0) {
244
0
0
                print STDERR "$0: gh program did not like these environment variables: ".join(', ', @problematic_env_variables)." -- consider unsetting them.\n";
245
0
0
                return 1;
246            }
247        }
248    }
249
250
1
30
    print $gh_status_lines;
251
1
12
    return 0;
252}
253
254sub tools_are_ready {
255
3
3
100759
    my ($program) = @_;
256
3
50
9
    unless (gh_is_happy($program)) {
257
1
4
        $! = 1;
258
1
50
33
6
        my $or_gh_token = (defined $ENV{CI} && $ENV{CI}) ? ' or set the GH_TOKEN environment variable' : '';
259
1
6
        die_custom 243, "$program requires a happy gh, please try 'gh auth login'$or_gh_token\n";
260    }
261}
262
263sub run_pipe {
264
1
1
5
    my @args = @_;
265
1
2
    my ($out, undef, $exit_code) = capture_system(@args);
266
1
6
    return $out;
267}
268
269sub unzip_pipe {
270
0
0
0
    my ($artifact, $file) = @_;
271
0
0
    return run_pipe(
272        'unzip',
273        '-p', $artifact,
274        $file
275    );
276}
277
278sub retrieve_spell_check_this {
279
0
0
0
    my ($artifact, $config_ref) = @_;
280
0
0
    my $spell_check_this_config = unzip_pipe($artifact, 'spell_check_this.json');
281
0
0
0
    return unless $spell_check_this_config =~ /\{.*\}/s;
282
0
0
    my %config;
283
0
0
0
0
0
0
0
    eval { %config = %{decode_json $spell_check_this_config}; } || die_custom 267, "decode_json failed in retrieve_spell_check_this with '$spell_check_this_config'";
284
0
0
    my ($repo, $branch, $destination, $path) = ($config{url}, $config{branch}, $config{config}, $config{path});
285
0
0
    my $spell_check_this_dir = tempdir();
286
0
0
    my $exit_code;
287
0
0
    (undef, undef, $exit_code) = capture_system(
288            'git', 'clone',
289            '--depth', '1',
290            '--no-tags',
291            $repo,
292            '--branch', $branch,
293            $spell_check_this_dir
294        );
295
0
0
0
    if ($?) {
296
0
0
        die_custom 280, "git clone $repo#$branch failed";
297    }
298
299
0
0
    make_path($destination);
300
0
0
    system('cp', '-i', '-R', glob("$spell_check_this_dir/$path/*"), $destination);
301
0
0
    system('git', 'add', '-f', $destination);
302}
303
304sub case_biased {
305
0
0
0
    lc($a)."-".$a cmp lc($b)."-".$b;
306}
307
308sub add_to_excludes {
309
0
0
0
    my ($artifact, $config_ref) = @_;
310
0
0
0
0
    my %config = %{$config_ref};
311
0
0
    my $excludes = $config{"excludes_file"};
312
0
0
    my $should_exclude_patterns = unzip_pipe($artifact, 'should_exclude.patterns');
313
0
0
0
    unless ($should_exclude_patterns =~ /\w/) {
314
0
0
        $should_exclude_patterns = unzip_pipe($artifact, 'should_exclude.txt');
315
0
0
0
        return unless $should_exclude_patterns =~ /\w/;
316
0
0
        $should_exclude_patterns =~ s{^(.*)}{^\\Q$1\\E\$}gm;
317    }
318
0
0
    my $need_to_add_excludes;
319    my %excludes;
320
0
0
0
    if (-f $excludes) {
321
0
0
        open EXCLUDES, '<', $excludes;
322
0
0
        while (<EXCLUDES>) {
323
0
0
            chomp;
324
0
0
0
            next unless /./;
325
0
0
            $excludes{$_."\n"} = 1;
326        }
327
0
0
        close EXCLUDES;
328    } else {
329
0
0
        $need_to_add_excludes = 1;
330    }
331
0
0
    for $pattern (split /\n/, $should_exclude_patterns) {
332
0
0
0
        next unless $pattern =~ /./;
333
0
0
        $excludes{$pattern."\n"} = 1;
334    }
335
0
0
    open EXCLUDES, '>', $excludes;
336
0
0
    print EXCLUDES join "", sort case_biased keys %excludes;
337
0
0
    close EXCLUDES;
338
0
0
0
    system('git', 'add', '--', $excludes) if $need_to_add_excludes;
339}
340
341sub remove_stale {
342
0
0
0
    my ($artifact, $config_ref) = @_;
343
0
0
    my @stale = split /\s+/s, unzip_pipe($artifact, 'remove_words.txt');
344
0
0
0
    return unless @stale;
345
0
0
0
0
    my %config = %{$config_ref};
346
0
0
0
0
    my @expect_files = @{$config{"expect_files"}};
347    @expect_files = grep {
348
0
0
0
0
0
        print STDERR "Could not find $_\n" unless -f $_;
349
0
0
        -f $_;
350    } @expect_files;
351
0
0
0
    unless (@expect_files) {
352
0
0
        die_custom 336, "Could not find any of the processed expect files, are you on the wrong branch?";
353    }
354
355
0
0
    my $re = join "|", @stale;
356
0
0
    for my $file (@expect_files) {
357
0
0
        open INPUT, '<', $file;
358
0
0
        my @keep;
359
0
0
        while (<INPUT>) {
360
0
0
0
            next if /^(?:$re)(?:(?:\r|\n)*$|[# ].*)/;
361
0
0
            push @keep, $_;
362        }
363
0
0
        close INPUT;
364
365
0
0
        open OUTPUT, '>', $file;
366
0
0
        print OUTPUT join '', @keep;
367
0
0
        close OUTPUT;
368    };
369}
370
371sub add_expect {
372
0
0
0
    my ($artifact, $config_ref) = @_;
373
0
0
    my @add = split /\s+/s, (unzip_pipe($artifact, 'tokens.txt'));
374
0
0
0
    return unless @add;
375
0
0
0
0
    my %config = %{$config_ref};
376
0
0
    my $new_expect_file = $config{"new_expect_file"};
377
0
0
    my @words;
378
0
0
    make_path (dirname($new_expect_file));
379
0
0
0
    if (-s $new_expect_file) {
380
0
0
        open FILE, q{<}, $new_expect_file;
381
0
0
        local $/ = undef;
382
0
0
        @words = split /\s+/, <FILE>;
383
0
0
        close FILE;
384    }
385
0
0
    my %items;
386
0
0
    @items{@words} = @words x (1);
387
0
0
    @items{@add} = @add x (1);
388
0
0
    @words = sort case_biased keys %items;
389
0
0
    open FILE, q{>}, $new_expect_file;
390
0
0
    for my $word (@words) {
391
0
0
0
        print FILE "$word\n" if $word =~ /\S/;
392    };
393
0
0
    close FILE;
394
0
0
    system("git", "add", $new_expect_file);
395}
396
397sub get_artifact_metadata {
398
3
3
4
    my ($url) = @_;
399
3
10
    my $json_file = tempfile_name();
400
3
13
    my ($curl_stdout, $curl_stderr, $curl_result);
401
3
10
    my @curl_args = (
402        'curl',
403        $url,
404        '-A',
405        $ua,
406        '-s',
407        '--fail-with-body',
408    );
409
3
5
    my ($gh_token) = get_token();
410
3
50
8
    push @curl_args, '-u', "token:$gh_token" if defined $gh_token;
411
3
3
    push @curl_args, (
412        '-o',
413        $json_file
414    );
415
3
11
    ($curl_stdout, $curl_stderr, $curl_result) = capture_system(
416        @curl_args
417    );
418
3
100
15
    unless ($curl_result == 0) {
419
1
50
6
        if ($curl_stdout eq '') {
420
1
5
            local $/;
421
1
28
            open my $error_fh, '<', $json_file;
422
1
12
            $curl_stdout = <$error_fh>;
423
1
5
            close $error_fh;
424        }
425        return (
426
1
12
            out    => $curl_stdout,
427            err    => $curl_stderr,
428            result => $curl_result,
429        );
430    }
431
2
6
    my $link;
432
2
80
    open my $json_file_fh, '<', $json_file;
433
2
3
    my ($id, $download_url, $count);
434    {
435
2
2
2
12
        local $/;
436
2
28
        my $content = <$json_file_fh>;
437
2
19
        my $json = decode_json $content;
438
2
5677
        my $artifact = $json->{'artifacts'}->[0];
439
2
3
        $id = $artifact->{'id'};
440
2
3
        $download_url = $artifact->{'archive_download_url'};
441
2
8
        $count = $json->{'total_count'};
442    }
443
2
9
    close $json_file_fh;
444
2
50
4
    if ($count == 0) {
445        return (
446
0
0
            out => '',
447            err => 'no artifact matches any of the names or patterns provided',
448            result => (3 << 8),
449        );
450    }
451    return (
452
2
16
        id       => $id,
453        download => $download_url,
454        count    => $count,
455    );
456}
457
458sub get_latest_artifact_metadata {
459
2
2
2
    my ($artifact_dir, $repo, $run, $artifact_name) = @_;
460
2
2
    my $page = 1;
461
2
4
    my $url = "$ENV{GITHUB_API_URL}/repos/$repo/actions/runs/$run/artifacts?name=$artifact_name&per_page=1&page=";
462
2
9
    my %first = get_artifact_metadata($url.$page);
463
2
5
    $page = $first{'count'};
464
2
100
5
    if (defined $page) {
465
1
2
        my %second = get_artifact_metadata($url.$page);
466
1
6
        my ($id_1, $id_2) = ($first{'id'}, $second{'id'});
467
1
50
33
9
        if (defined $id_1 && defined $id_2) {
468
1
50
3
            if ($id_2 > $id_1) {
469                return (
470
0
0
                    download => $second{'download'},
471                );
472            }
473        }
474    }
475
2
4
    my $download = $first{'download'};
476
2
100
3
    if (defined $download) {
477        return (
478
1
4
            download => $download,
479        );
480    }
481
1
7
    return %first;
482}
483
484sub download_latest_artifact {
485
2
2
6
    my %maybe_download = get_latest_artifact_metadata(@_);
486
2
4
    my $download = $maybe_download{'download'};
487
2
6
    my $zip_file = tempfile_name();
488
2
100
6
    if (defined $download) {
489
1
3
        my @curl_args = (
490            'curl',
491            $download,
492            '-L',
493            '-A',
494            $ua,
495            '-s',
496            '--fail-with-body',
497        );
498
1
14
        my ($gh_token) = get_token();
499
1
50
2
        push @curl_args, '-u', "token:$gh_token" if defined $gh_token;
500
1
2
        push @curl_args, (
501            '-o',
502            $zip_file
503        );
504
1
1
        ($curl_stdout, $curl_stderr, $curl_result) = capture_system(
505            @curl_args
506        );
507
1
50
7
        if ($curl_result != 0) {
508
1
50
13
            if ($curl_stdout eq '') {
509
1
10
                local $/;
510
1
32
                open my $error_fh, '<', $zip_file;
511
1
10
                $curl_stdout = <$error_fh>;
512
1
5
                close $error_fh;
513            }
514
1
10
            return ("$curl_stdout\n$curl_stderr", $curl_result);
515        }
516
0
0
        my ($artifact_dir, $repo, $run, $artifact_name) = @_;
517
0
0
        ($out, $err, $result) = capture_system(
518            'unzip',
519            '-q',
520            $zip_file,
521            '-d',
522            $artifact_dir,
523            );
524
0
0
        return ("$out\n$err", $result);
525    }
526
1
2
    my ($out, $err, $result) = ($maybe_download{'out'}, $maybe_download{'err'}, $maybe_download{'result'});
527
1
5
    return ("$out\n$err", $result);
528}
529
530sub get_artifacts {
531
2
2
2854
    my ($repo, $run, $suffix) = @_;
532
2
1
    our $program;
533
2
7
    my $artifact_dir = tempdir(CLEANUP => 1);
534
2
399
    my $gh_err_text;
535
2
2
    my $artifact_name = 'check-spelling-comment';
536
2
50
4
    if ($suffix) {
537
0
0
        $artifact_name .= "-$suffix";
538    }
539
2
2
    my $retries_remaining = 3;
540
2
12
    while ($retries_remaining-- > 0) {
541
2
8
        ($gh_err_text, $ret) = download_latest_artifact(
542            $artifact_dir,
543            $repo,
544            $run,
545            $artifact_name
546        );
547
2
50
7
        return glob("$artifact_dir/artifact*.zip") unless ($ret >> 8);
548
549
2
7
        die_with_message($gh_err_text);
550
2
100
22
        if ($gh_err_text =~ /no valid artifacts found to download|"Artifact has expired"/) {
551
1
5
            my $expired_json = run_pipe(
552                'gh', 'api',
553                "/repos/$repo/actions/runs/$run/artifacts",
554                '-q',
555                '.artifacts.[]|select(.name=="'.$artifact_name.'")|.expired'
556            );
557
1
50
7
            if ($expired_json ne '') {
558
1
3
                chomp $expired_json;
559
1
1
                my $expired;
560
1
1
50
4
8
                eval { $expired = decode_json $expired_json } || die_custom 544, "decode_json failed in update_repository with '$expired_json'";
561
1
50
130
                if ($expired) {
562
1
33
                    print "$program: GitHub Run Artifact expired. You will need to trigger a new run.\n";
563
1
1
5
2
                    tear_here(1); die_custom 547, "exiting";
564                }
565            }
566
0
0
            print "$program: GitHub Run may not have completed. If so, please wait for it to finish and try again.\n";
567
0
0
0
0
            tear_here(2); die_custom 551, "exiting";
568        }
569
1
50
3
        if ($gh_err_text =~ /no artifact matches any of the names or patterns provided/) {
570
0
0
0
            $github_server_url = $ENV{GITHUB_SERVER_URL} || '';
571
0
0
            my $run_link;
572
0
0
0
            if ($github_server_url) {
573
0
0
                $run_link = "[$run]($github_server_url/$repo/actions/runs/$run)";
574            } else {
575
0
0
                $run_link = "$run";
576            }
577
0
0
            print "$program: The referenced repository ($repo) run ($run_link) does not have a corresponding artifact ($artifact_name). If it was deleted, that's unfortunate. Consider pushing a change to the branch to trigger a new run?\n";
578
0
0
            print "If you don't think anyone deleted the artifact, please file a bug to https://github.com/check-spelling/check-spelling/issues/new including as much information about how you triggered this error as possible.\n";
579
0
0
0
0
            tear_here(3); die_custom 563, "exiting";
580        }
581
1
50
7
        if ($gh_err_text =~ /HTTP 404: Not Found|"status":"404"/) {
582
1
22
            print "$program: The referenced repository ($repo) may not exist, perhaps you do not have permission to see it. If the repository is hosted by GitHub Enterprise, check-spelling does not know how to integrate with it.\n";
583
1
1
6
3
            tear_here(8); die_custom 567, "exiting";
584        }
585
0
0
0
        if ($gh_err_text =~ /HTTP 403: API rate limit exceeded for .*?./) {
586        } elsif ($gh_err_text =~ m{dial tcp \S+:\d+: i/o timeout$}) {
587
0
0
            if ($retries_remaining <= 0) {
588
0
                print "$program: Timeout connecting to GitHub. This is probably caused by an outage of sorts.\nCheck https://www.githubstatus.com/history\nTry again later.";
589
0
0
                tear_here(9); die_custom 573, "exiting";
590            }
591        } else {
592
0
            print "$program: Unknown error, please check the list of known issues https://github.com/check-spelling/check-spelling/issues?q=is%3Aissue%20apply.pl and file a bug to https://github.com/check-spelling/check-spelling/issues/new?title=%60apply.pl%60%20scenario&body=Please%20provide%20details+preferably%20including%20a%20link%20to%20a%20workflow%20run,%20the%20configuration%20of%20the%20repository,%20and%20anything%20else%20you%20may%20know%20about%20the%20problem%2e\n";
593
0
            print $gh_err_text;
594
0
0
            tear_here(4); die_custom 578, "exiting";
595        }
596
0
0
        my $request_id = $1 if ($gh_err_text =~ /\brequest ID\s+(\S+)/);
597
0
0
        my $timestamp = $1 if ($gh_err_text =~ /\btimestamp\s+(.*? UTC)/);
598
0
0
        my $has_gh_token = defined $ENV{GH_TOKEN} || defined $ENV{GITHUB_TOKEN};
599
0
        my $meta_url = 'https://api.github.com/meta';
600
0
        while (1) {
601
0
            my @curl_args = qw(curl);
602
0
0
            unless ($has_gh_token) {
603
0
                my ($gh_token) = get_token();
604
0
0
                push @curl_args, '-u', "token:$gh_token" if defined $gh_token;
605            }
606
0
            push @curl_args, '-I', $meta_url;
607
0
            my ($curl_stdout, $curl_stderr, $curl_result);
608
0
            ($curl_stdout, $curl_stderr, $curl_result) = capture_system(@curl_args);
609
0
            my $delay = 1;
610
0
0
0
            if ($curl_stdout =~ m{^HTTP/\S+\s+200}) {
611
0
0
                if ($curl_stdout =~ m{^x-ratelimit-remaining:\s+(\d+)$}m) {
612
0
                    my $ratelimit_remaining = $1;
613
0
0
                    last if ($ratelimit_remaining > 10);
614
615
0
                    $delay = 5;
616
0
                    print STDERR "Sleeping for $delay seconds because $ratelimit_remaining is close to 0\n";
617                } else {
618
0
                    print STDERR "Couldn't find x-ratelimit-remaining, will sleep for $delay\n";
619                }
620            } elsif ($curl_stdout =~ m{^HTTP/\S+\s+403}) {
621
0
0
                if ($curl_stdout =~ /^retry-after:\s+(\d+)/m) {
622
0
                    $delay = $1;
623
0
                    print STDERR "Sleeping for $delay seconds (presumably due to API rate limit)\n";
624                } else {
625
0
                    print STDERR "Couldn't find retry-after, will sleep for $delay\n";
626                }
627            } else {
628
0
0
                my $response = $1 if $curl_stdout =~ m{^(HTTP/\S+)};
629
0
                print STDERR "Unexpected response ($response) from $meta_url; sleeping for $delay\n";
630            }
631
0
            sleep $delay;
632        }
633    }
634}
635
636sub update_repository {
637
0
0
    my ($artifact) = @_;
638
0
0
    die_custom 622, if $artifact =~ /'/;
639
0
    our $program;
640
0
    my $apply = unzip_pipe($artifact, 'apply.json');
641
0
0
    unless ($apply =~ /\{.*\}/s) {
642
0
        print STDERR "$program: Could not retrieve valid apply.json from artifact\n";
643
0
        $apply = '{
644            "expect_files": [".github/actions/spelling/expect.txt"],
645            "new_expect_file": ".github/actions/spelling/expect.txt",
646            "excludes_file": ".github/actions/spelling/excludes.txt",
647            "spelling_config": ".github/actions/spelling"
648        }';
649    }
650
0
    my $config_ref;
651
0
0
0
    eval { $config_ref = decode_json($apply); } ||
652        die_custom 636, "$program: decode_json failed in update_repository with '$apply'";
653
654
0
    my $git_repo_root = run_pipe('git', 'rev-parse', '--show-toplevel');
655
0
    chomp $git_repo_root;
656
0
0
    die_custom 640, "$program: Could not find git repo root..." unless $git_repo_root =~ /\w/;
657
0
    chdir $git_repo_root;
658
659
0
    retrieve_spell_check_this($artifact, $config_ref);
660
0
    remove_stale($artifact, $config_ref);
661
0
    add_expect($artifact, $config_ref);
662
0
    add_to_excludes($artifact, $config_ref);
663
0
    system('git', 'add', '-u', '--', $config_ref->{'spelling_config'});
664}
665
666sub extract_artifacts_from_file {
667
0
0
    my ($artifact) = @_;
668
0
    open my $artifact_reader, '-|', 'unzip', '-l', $artifact;
669
0
    my ($has_artifact, $only_file) = (0, 0);
670
0
    while (my $line = <$artifact_reader>) {
671
0
        chomp $line;
672
0
0
        if ($line =~ /\s+artifact\.zip$/) {
673
0
            $has_artifact = 1;
674
0
            next;
675        }
676
0
0
        if ($line =~ /\s+1 file$/) {
677
0
            $only_file = 1;
678
0
            next;
679        }
680
0
0
        $only_file = 0 if $only_file;
681    }
682
0
    close $artifact_reader;
683
0
    my @artifacts;
684
0
0
0
    if ($has_artifact && $only_file) {
685
0
        my $artifact_dir = tempdir(CLEANUP => 1);
686
0
        my ($fh, $gh_err) = tempfile();
687
0
        close $fh;
688
0
        system('unzip', '-q', '-d', $artifact_dir, $artifact, 'artifact.zip');
689
0
        @artifacts = ("$artifact_dir/artifact.zip");
690    } else {
691
0
        @artifacts = ($artifact);
692    }
693
0
    return @artifacts;
694}
695
696sub main {
697
0
0
    our $program;
698
0
    my ($bash_script, $first, $run);
699
0
    ($program, $bash_script, $first, $run) = @_;
700
0
    my $syntax = "$program <RUN_URL | OWNER/REPO RUN | ARTIFACT.zip>";
701    # Stages
702    # - 1 check for tools basic
703
0
    check_basic_tools();
704    # - 2 check for current
705    # -> 1. download the latest version to a temp file
706    # -> 2. parse current and latest (stripping comments) and compare (whitespace insensitively)
707    # -> 3. offer to update if the latest version is different
708
0
    check_current_script($bash_script);
709    # - 4 parse arguments
710
0
0
    die_custom 694, $syntax unless defined $first;
711
0
0
    $ENV{'GITHUB_API_URL'} ||= 'https://api.github.com';
712
0
    my $repo;
713    my @artifacts;
714
0
0
    if (-s $first) {
715
0
        @artifacts = extract_artifacts_from_file($first);
716    } else {
717
0
        my $suffix;
718
0
0
        if ($first =~ m{^\s*https://.*/([^/]+/[^/]+)/actions/runs/(\d+)(?:/attempts/\d+|)(?:#(\S+)|)\s*$}) {
719
0
            ($repo, $run, $suffix) = ($1, $2, $3);
720        } else {
721
0
            $repo = $first;
722        }
723
0
0
0
        die_custom 707, $syntax unless defined $repo && defined $run;
724        # - 3 check for tool readiness (is `gh` working)
725
0
        tools_are_ready($program);
726
0
        @artifacts = get_artifacts($repo, $run, $suffix);
727    }
728
729    # - 5 do work
730
0
    for my $artifact (@artifacts) {
731
0
        update_repository($artifact);
732    }
733}
734