File Coverage

File:lib/CheckSpelling/Apply.pm
Coverage:46.4%

linestmtbrancondsubtimecode
1package CheckSpelling::Apply;
2sub tear_here {
3
8
8
20
  my ($exit) = @_;
4
8
6
  our $exited;
5
8
100
39
  return if defined $exited;
6
6
69
  print STDERR "\n<<<TEAR HERE<<<exit: $exit\n";
7
6
28
  print STDOUT "\n<<<TEAR HERE<<<exit: $exit\n";
8
6
24
  $exited = $exit;
9}
10sub die_custom {
11
4
4
10
  my ($line, $message) = @_;
12
4
4
  our $program;
13
4
46
  print STDERR "$message at $program line $line.\n";
14
4
13
  tear_here(1);
15}
16#!/usr/bin/env perl
17":" || q@<<"=END_OF_PERL"@;
18
19
1
1
1
1
400789
1
45
use Symbol 'gensym';
20
1
1
1
1
186
1278
29
use IPC::Open3;
21
1
1
1
1
4
1
27
use File::Basename qw(dirname);
22
1
1
1
1
2
1
18
use File::Path qw(make_path);
23
1
1
1
1
173
319
31
use File::Spec::Functions qw(catfile path);
24
1
1
1
1
2
1
18
use File::Temp qw/ tempfile tempdir /;
25
1
1
1
1
1
0
28
use JSON::PP;
26
1
1
1
1
2
1
2498
use warnings;
27
28my @safe_path = qw(
29    /opt/homebrew/bin
30    /opt/homebrew/sbin
31    /usr/local/bin
32    /usr/bin
33    /bin
34    /usr/sbin
35    /sbin
36);
37
38my $bin = glob("~/bin");
39push @safe_path, $bin if -d $bin;
40
41my $ua = 'check-spelling-agent/0.0.4';
42
43$ENV{'PATH'} = join ':', @safe_path unless defined $ENV{SYSTEMROOT};
44
45sub check_exists_command {
46
6
6
5
    my ($program) = @_;
47
48
6
7
    my @path = path;
49
6
45
    my @pathext = ('');
50
51
6
50
8
    if ($^O eq 'MSWin32') {
52
0
0
0
0
        push @pathext, map { lc } split /;/, $ENV{PATHEXT};
53    }
54
55
6
5
    for my $dir (@path) {
56
27
9
        for my $suffix (@pathext) {
57
27
38
            my $f = catfile $dir, "$program$suffix";
58
27
100
124
            return $f if -x $f;
59        }
60    }
61}
62
63sub needs_command_because {
64
5
5
1933
    my ($program, $reason) = @_;
65
5
100
5
    return if check_exists_command($program);
66
1
5
    die_custom 51, 'Please install `'.$program.'` - it is needed to '.$reason;
67}
68
69sub check_basic_tools {
70
1
1
1289
    needs_command_because('git', 'interact with git repositories');
71
1
2
    needs_command_because('curl', 'download other tools');
72
1
1
    needs_command_because('gh', 'interact with github');
73    #needs_command_because('magic-magic', 'debugging');
74}
75
76sub get_token {
77
4
4
4
    our $token;
78
4
100
67
20
    return $token if defined $token && $token ne '';
79
1
33
3
    $token = $ENV{'GH_TOKEN'} || $ENV{'GITHUB_TOKEN'};
80
1
50
33
4
    return $token if defined $token && $token ne '';
81
0
0
    $token = `gh auth token`;
82
0
0
    chomp $token;
83
0
0
    return $token;
84};
85
86sub download_with_curl {
87
1
1
2
    my ($url, $dest, $flags) = @_;
88
1
50
4
    $flags = '-fsL' unless defined $flags;
89
1
90424
    system('curl',
90        '--connect-timeout', 3,
91        '-A', $ua,
92        $flags,
93        '-o', $dest,
94        $url
95    );
96}
97
98sub tempfile_name {
99
6
6
25
    my ($fh, $filename) = tempfile();
100
6
1246
    close $fh;
101
6
12
    return $filename;
102}
103
104sub strip_comments {
105
4
4
5
    my ($file) = @_;
106
4
15
    my ($fh, $filename) = tempfile();
107
4
812
    open INPUT, '<', $file;
108
4
53
    while (<INPUT>) {
109
1842
100
979
        next if /^\s*(?:#.*)/;
110
1814
1109
        print $fh $_;
111    }
112
4
11
    close INPUT;
113
4
50
    close $fh;
114
4
9
    return $filename;
115}
116
117sub capture_system {
118
21
21
57
    my @args = @_;
119
21
96
    my $pid = open3(my $child_in, my $child_out, my $child_err = gensym, @args);
120
21
40858
    my (@err, @out);
121
21
2150275
    while (my $output = <$child_out>) {
122
2
1706
        push @out, $output;
123    }
124
21
504
    while (my $error = <$child_err>) {
125
24
100
        push @err, $error;
126    }
127
21
352
    waitpid( $pid, 0 );
128
21
104
    my $child_exit_status = $?;
129
21
72
    my $output_joined = join '', @out;
130
21
50
    my $error_joined = join '', @err;
131
21
722
    return ($output_joined, $error_joined, $child_exit_status);
132}
133
134sub capture_merged_system {
135
14
14
36
    my ($output_joined, $error_joined, $child_exit_status) = capture_system(@_);
136
14
50
67
    my $joiner = ($output_joined ne '') ? "\n" : '';
137
14
77
    return ($output_joined.$joiner.$error_joined, $child_exit_status);
138}
139
140sub compare_files {
141
2
2
1493
    my ($one, $two) = @_;
142
2
6
    my $one_stripped = strip_comments($one);
143
2
2
    my $two_stripped = strip_comments($two);
144
2
1
    my $exit_code;
145
2
5
    (undef, undef, $exit_code) = capture_system(
146            'diff',
147            '-qwB',
148            $one_stripped, $two_stripped
149        );
150
2
50
4
    if ($? == -1) {
151
0
0
        print "could not compare '$one' and '$two': $!\n";
152
0
0
        return 0;
153    }
154
2
50
9
    if ($? & 127) {
155
0
0
0
        printf "child died with signal %d, %s core dump\n",
156        ($? & 127),  ($? & 128) ? 'with' : 'without';
157
0
0
        return 0;
158    }
159
2
100
10
    return 0 if $? == 0;
160
1
6
    return 1;
161}
162
163my $bash_script=q{
164=END_OF_PERL@
165# bash
166set -e
167if [ "$OUTPUT" = "$ERROR" ]; then
168    ("$@" 2>&1) > "$OUTPUT"
169else
170    "$@" > "$OUTPUT" 2> "$ERROR"
171fi
172exit
173};
174
175sub check_current_script {
176
1
50
1
121165
    if ("$0" eq '-') {
177
0
0
        my ($bash_script) = @_;
178
0
0
        my $fh;
179
0
0
        ($fh, $0) = tempfile();
180
0
0
        $bash_script =~ s/^=.*\@$//m;
181
0
0
        print $fh $bash_script;
182
0
0
        close $fh;
183
0
0
        return;
184    }
185
1
4
    my $filename = tempfile_name();
186
1
1
    my $source = 'https://raw.githubusercontent.com/check-spelling/check-spelling/prerelease/apply.pl';
187
1
3
    download_with_curl($source, $filename);
188
1
50
23
    if ($? == 0) {
189
1
50
22
        if (compare_files($filename, $0)) {
190
0
0
            print "Current apply script differs from '$source' (locally downloaded to `$filename`). You may wish to upgrade.\n";
191        }
192    }
193}
194
195sub die_with_message {
196
5
5
6
    our $program;
197
5
9
    my ($gh_err_text) = @_;
198
5
50
33
25
    if ($gh_err_text =~ /error connecting to / && $gh_err_text =~ /check your internet connection/) {
199
0
0
        print "$program: Internet access may be limited. Check your connection (this often happens with lousy cable internet service providers where their CG-NAT or whatever strands the modem).\n\n$gh_err_text";
200
0
0
0
0
        tear_here(5); return -1000;
201    }
202
5
100
20
    if ($gh_err_text =~ /proxyconnect tcp:.*connect: connection refused/) {
203
1
31
        print "$program: Proxy is not accepting connections.\n";
204
1
4
        for my $proxy (qw(http_proxy HTTP_PROXY https_proxy HTTPS_PROXY)) {
205
4
100
10
            if (defined $ENV{$proxy}) {
206
1
5
                print "  $proxy: '$ENV{$proxy}'\n";
207            }
208        }
209
1
4
        print "\n$gh_err_text";
210
1
1
5
1
        tear_here(6); return -1000;
211    }
212
4
100
17
    if ($gh_err_text =~ /dial unix .*: connect: .*/) {
213
1
33
        print "$program: Unix http socket is not working.\n";
214
1
29988
        my $gh_http_unix_socket = `gh config get http_unix_socket`;
215
1
31
        print "  http_unix_socket: $gh_http_unix_socket\n";
216
1
6
        print "\n$gh_err_text";
217
1
1
13
6
        tear_here(7); return -1000;
218    }
219}
220
221sub gh_is_happy_internal {
222
7
7
22
    my ($output, $exit_code) = capture_merged_system(qw(gh api /installation/repositories));
223
7
50
21
    return ($exit_code, $output) if $exit_code == 0;
224
7
16
    ($output, $exit_code) = capture_merged_system(qw(gh api /user));
225
7
20
    return ($exit_code, $output);
226}
227
228sub gh_is_happy {
229
3
3
5
    my ($program) = @_;
230
3
6
    my ($gh_auth_status, $gh_status_lines) = gh_is_happy_internal();
231
3
50
29
    return 1 if $gh_auth_status == 0;
232
3
12
    die_with_message($gh_status_lines);
233
234
3
4
    my @problematic_env_variables;
235
3
6
    for my $variable (qw(GH_TOKEN GITHUB_TOKEN GITHUB_ACTIONS CI)) {
236
12
100
27
        if (defined $ENV{$variable}) {
237
4
51
            delete $ENV{$variable};
238
4
8
            push @problematic_env_variables, $variable;
239
4
8
            ($gh_auth_status, $gh_status_lines) = gh_is_happy_internal();
240
4
50
22
            if ($gh_auth_status == 0) {
241
0
0
                print STDERR "$0: gh program did not like these environment variables: ".join(', ', @problematic_env_variables)." -- consider unsetting them.\n";
242
0
0
                return 1;
243            }
244        }
245    }
246
247
3
79
    print $gh_status_lines;
248
3
27
    return 0;
249}
250
251sub tools_are_ready {
252
3
3
102433
    my ($program) = @_;
253
3
50
12
    unless (gh_is_happy($program)) {
254
3
11
        $! = 1;
255
3
50
33
14
        my $or_gh_token = (defined $ENV{CI} && $ENV{CI}) ? ' or set the GH_TOKEN environment variable' : '';
256
3
17
        die_custom 241, "$program requires a happy gh, please try 'gh auth login'$or_gh_token\n";
257    }
258}
259
260sub run_pipe {
261
1
1
4
    my @args = @_;
262
1
1
    my ($out, undef, $exit_code) = capture_system(@args);
263
1
8
    return $out;
264}
265
266sub unzip_pipe {
267
0
0
0
    my ($artifact, $file) = @_;
268
0
0
    return run_pipe(
269        'unzip',
270        '-p', $artifact,
271        $file
272    );
273}
274
275sub retrieve_spell_check_this {
276
0
0
0
    my ($artifact, $config_ref) = @_;
277
0
0
    my $spell_check_this_config = unzip_pipe($artifact, 'spell_check_this.json');
278
0
0
0
    return unless $spell_check_this_config =~ /\{.*\}/s;
279
0
0
    my %config;
280
0
0
0
0
0
0
0
    eval { %config = %{decode_json $spell_check_this_config}; } || die_custom 265, "decode_json failed in retrieve_spell_check_this with '$spell_check_this_config'";
281
0
0
    my ($repo, $branch, $destination, $path) = ($config{url}, $config{branch}, $config{config}, $config{path});
282
0
0
    my $spell_check_this_dir = tempdir();
283
0
0
    my $exit_code;
284
0
0
    (undef, undef, $exit_code) = capture_system(
285            'git', 'clone',
286            '--depth', '1',
287            '--no-tags',
288            $repo,
289            '--branch', $branch,
290            $spell_check_this_dir
291        );
292
0
0
0
    if ($?) {
293
0
0
        die_custom 278, "git clone $repo#$branch failed";
294    }
295
296
0
0
    make_path($destination);
297
0
0
    system('cp', '-i', '-R', glob("$spell_check_this_dir/$path/*"), $destination);
298
0
0
    system('git', 'add', '-f', $destination);
299}
300
301sub case_biased {
302
0
0
0
    lc($a)."-".$a cmp lc($b)."-".$b;
303}
304
305sub add_to_excludes {
306
0
0
0
    my ($artifact, $config_ref) = @_;
307
0
0
0
0
    my %config = %{$config_ref};
308
0
0
    my $excludes = $config{"excludes_file"};
309
0
0
    my $should_exclude_patterns = unzip_pipe($artifact, 'should_exclude.patterns');
310
0
0
0
    unless ($should_exclude_patterns =~ /\w/) {
311
0
0
        $should_exclude_patterns = unzip_pipe($artifact, 'should_exclude.txt');
312
0
0
0
        return unless $should_exclude_patterns =~ /\w/;
313
0
0
        $should_exclude_patterns =~ s{^(.*)}{^\\Q$1\\E\$}gm;
314    }
315
0
0
    my $need_to_add_excludes;
316    my %excludes;
317
0
0
0
    if (-f $excludes) {
318
0
0
        open EXCLUDES, '<', $excludes;
319
0
0
        while (<EXCLUDES>) {
320
0
0
            chomp;
321
0
0
0
            next unless /./;
322
0
0
            $excludes{$_."\n"} = 1;
323        }
324
0
0
        close EXCLUDES;
325    } else {
326
0
0
        $need_to_add_excludes = 1;
327    }
328
0
0
    for $pattern (split /\n/, $should_exclude_patterns) {
329
0
0
0
        next unless $pattern =~ /./;
330
0
0
        $excludes{$pattern."\n"} = 1;
331    }
332
0
0
    open EXCLUDES, '>', $excludes;
333
0
0
    print EXCLUDES join "", sort case_biased keys %excludes;
334
0
0
    close EXCLUDES;
335
0
0
0
    system('git', 'add', '--', $excludes) if $need_to_add_excludes;
336}
337
338sub remove_stale {
339
0
0
0
    my ($artifact, $config_ref) = @_;
340
0
0
    my @stale = split /\s+/s, unzip_pipe($artifact, 'remove_words.txt');
341
0
0
0
    return unless @stale;
342
0
0
0
0
    my %config = %{$config_ref};
343
0
0
0
0
    my @expect_files = @{$config{"expect_files"}};
344    @expect_files = grep {
345
0
0
0
0
0
        print STDERR "Could not find $_\n" unless -f $_;
346
0
0
        -f $_;
347    } @expect_files;
348
0
0
0
    unless (@expect_files) {
349
0
0
        die_custom 334, "Could not find any of the processed expect files, are you on the wrong branch?";
350    }
351
352
0
0
    my $re = join "|", @stale;
353
0
0
    for my $file (@expect_files) {
354
0
0
        open INPUT, '<', $file;
355
0
0
        my @keep;
356
0
0
        while (<INPUT>) {
357
0
0
0
            next if /^(?:$re)(?:(?:\r|\n)*$|[# ].*)/;
358
0
0
            push @keep, $_;
359        }
360
0
0
        close INPUT;
361
362
0
0
        open OUTPUT, '>', $file;
363
0
0
        print OUTPUT join '', @keep;
364
0
0
        close OUTPUT;
365    };
366}
367
368sub add_expect {
369
0
0
0
    my ($artifact, $config_ref) = @_;
370
0
0
    my @add = split /\s+/s, (unzip_pipe($artifact, 'tokens.txt'));
371
0
0
0
    return unless @add;
372
0
0
0
0
    my %config = %{$config_ref};
373
0
0
    my $new_expect_file = $config{"new_expect_file"};
374
0
0
    my @words;
375
0
0
    make_path (dirname($new_expect_file));
376
0
0
0
    if (-s $new_expect_file) {
377
0
0
        open FILE, q{<}, $new_expect_file;
378
0
0
        local $/ = undef;
379
0
0
        @words = split /\s+/, <FILE>;
380
0
0
        close FILE;
381    }
382
0
0
    my %items;
383
0
0
    @items{@words} = @words x (1);
384
0
0
    @items{@add} = @add x (1);
385
0
0
    @words = sort case_biased keys %items;
386
0
0
    open FILE, q{>}, $new_expect_file;
387
0
0
    for my $word (@words) {
388
0
0
0
        print FILE "$word\n" if $word =~ /\S/;
389    };
390
0
0
    close FILE;
391
0
0
    system("git", "add", $new_expect_file);
392}
393
394sub get_artifact_metadata {
395
3
3
3
    my ($url) = @_;
396
3
7
    my $json_file = tempfile_name();
397
3
2
    my ($curl_stdout, $curl_stderr, $curl_result);
398
3
12
    my @curl_args = (
399        'curl',
400        $url,
401        '-A',
402        $ua,
403        '-s',
404        '--fail-with-body',
405    );
406
3
6
    my $gh_token = get_token();
407
3
50
8
    push @curl_args, '-u', "token:$gh_token" if defined $gh_token;
408
3
3
    push @curl_args, (
409        '-o',
410        $json_file
411    );
412
3
6
    ($curl_stdout, $curl_stderr, $curl_result) = capture_system(
413        @curl_args
414    );
415
3
100
11
    unless ($curl_result == 0) {
416
1
50
8
        if ($curl_stdout eq '') {
417
1
9
            local $/;
418
1
31
            open my $error_fh, '<', $json_file;
419
1
11
            $curl_stdout = <$error_fh>;
420
1
4
            close $error_fh;
421        }
422        return (
423
1
15
            out    => $curl_stdout,
424            err    => $curl_stderr,
425            result => $curl_result,
426        );
427    }
428
2
3
    my $link;
429
2
64
    open my $json_file_fh, '<', $json_file;
430
2
4
    my ($id, $download_url, $count);
431    {
432
2
2
3
12
        local $/;
433
2
56
        my $content = <$json_file_fh>;
434
2
33
        my $json = decode_json $content;
435
2
5233
        my $artifact = $json->{'artifacts'}->[0];
436
2
2
        $id = $artifact->{'id'};
437
2
2
        $download_url = $artifact->{'archive_download_url'};
438
2
8
        $count = $json->{'total_count'};
439    }
440
2
7
    close $json_file_fh;
441
2
50
3
    if ($count == 0) {
442        return (
443
0
0
            out => '',
444            err => 'no artifact matches any of the names or patterns provided',
445            result => (3 << 8),
446        );
447    }
448    return (
449
2
18
        id       => $id,
450        download => $download_url,
451        count    => $count,
452    );
453}
454
455sub get_latest_artifact_metadata {
456
2
2
5
    my ($artifact_dir, $repo, $run, $artifact_name) = @_;
457
2
1
    my $page = 1;
458
2
3
    my $url = "$ENV{GITHUB_API_URL}/repos/$repo/actions/runs/$run/artifacts?name=$artifact_name&per_page=1&page=";
459
2
7
    my %first = get_artifact_metadata($url.$page);
460
2
6
    $page = $first{'count'};
461
2
100
4
    if (defined $page) {
462
1
5
        my %second = get_artifact_metadata($url.$page);
463
1
4
        my ($id_1, $id_2) = ($first{'id'}, $second{'id'});
464
1
50
33
9
        if (defined $id_1 && defined $id_2) {
465
1
50
4
            if ($id_2 > $id_1) {
466                return (
467
0
0
                    download => $second{'download'},
468                );
469            }
470        }
471    }
472
2
3
    my $download = $first{'download'};
473
2
100
4
    if (defined $download) {
474        return (
475
1
6
            download => $download,
476        );
477    }
478
1
8
    return %first;
479}
480
481sub download_latest_artifact {
482
2
2
6
    my %maybe_download = get_latest_artifact_metadata(@_);
483
2
3
    my $download = $maybe_download{'download'};
484
2
9
    my $zip_file = tempfile_name();
485
2
100
4
    if (defined $download) {
486
1
3
        my @curl_args = (
487            'curl',
488            $download,
489            '-L',
490            '-A',
491            $ua,
492            '-s',
493            '--fail-with-body',
494        );
495
1
3
        my $gh_token = get_token();
496
1
50
2
        push @curl_args, '-u', "token:$gh_token" if defined $gh_token;
497
1
2
        push @curl_args, (
498            '-o',
499            $zip_file
500        );
501
1
1
        ($curl_stdout, $curl_stderr, $curl_result) = capture_system(
502            @curl_args
503        );
504
1
50
6
        if ($curl_result != 0) {
505
1
50
12
            if ($curl_stdout eq '') {
506
1
6
                local $/;
507
1
32
                open my $error_fh, '<', $zip_file;
508
1
9
                $curl_stdout = <$error_fh>;
509
1
5
                close $error_fh;
510            }
511
1
12
            return ("$curl_stdout\n$curl_stderr", $curl_result);
512        }
513
0
0
        my ($artifact_dir, $repo, $run, $artifact_name) = @_;
514
0
0
        ($out, $err, $result) = capture_system(
515            'unzip',
516            '-q',
517            $zip_file,
518            '-d',
519            $artifact_dir,
520            );
521
0
0
        return ("$out\n$err", $result);
522    }
523
1
2
    my ($out, $err, $result) = ($maybe_download{'out'}, $maybe_download{'err'}, $maybe_download{'result'});
524
1
4
    return ("$out\n$err", $result);
525}
526
527sub get_artifacts {
528
2
2
3267
    my ($repo, $run, $suffix) = @_;
529
2
2
    our $program;
530
2
4
    my $artifact_dir = tempdir(CLEANUP => 1);
531
2
330
    my $gh_err_text;
532
2
3
    my $artifact_name = 'check-spelling-comment';
533
2
50
2
    if ($suffix) {
534
0
0
        $artifact_name .= "-$suffix";
535    }
536
2
2
    my $retries_remaining = 3;
537
2
6
    while ($retries_remaining-- > 0) {
538
2
4
        ($gh_err_text, $ret) = download_latest_artifact(
539            $artifact_dir,
540            $repo,
541            $run,
542            $artifact_name
543        );
544
2
50
7
        return glob("$artifact_dir/artifact*.zip") unless ($ret >> 8);
545
546
2
8
        die_with_message($gh_err_text);
547
2
100
20
        if ($gh_err_text =~ /no valid artifacts found to download|"Artifact has expired"/) {
548
1
12
            my $expired_json = run_pipe(
549                'gh', 'api',
550                "/repos/$repo/actions/runs/$run/artifacts",
551                '-q',
552                '.artifacts.[]|select(.name=="'.$artifact_name.'")|.expired'
553            );
554
1
50
6
            if ($expired_json ne '') {
555
1
2
                chomp $expired_json;
556
1
2
                my $expired;
557
1
1
50
2
10
                eval { $expired = decode_json $expired_json } || die_custom 542, "decode_json failed in update_repository with '$expired_json'";
558
1
50
123
                if ($expired) {
559
1
32
                    print "$program: GitHub Run Artifact expired. You will need to trigger a new run.\n";
560
1
1
5
14
                    tear_here(1); return -1000;
561                }
562            }
563
0
0
            print "$program: GitHub Run may not have completed. If so, please wait for it to finish and try again.\n";
564
0
0
0
0
            tear_here(2); return -1000;
565        }
566
1
50
2
        if ($gh_err_text =~ /no artifact matches any of the names or patterns provided/) {
567
0
0
0
            $github_server_url = $ENV{GITHUB_SERVER_URL} || '';
568
0
0
            my $run_link;
569
0
0
0
            if ($github_server_url) {
570
0
0
                $run_link = "[$run]($github_server_url/$repo/actions/runs/$run)";
571            } else {
572
0
0
                $run_link = "$run";
573            }
574
0
0
            print "$program: The referenced repository ($repo) run ($run_link) does not have a corresponding artifact ($artifact_name). If it was deleted, that's unfortunate. Consider pushing a change to the branch to trigger a new run?\n";
575
0
0
            print "If you don't think anyone deleted the artifact, please file a bug to https://github.com/check-spelling/check-spelling/issues/new including as much information about how you triggered this error as possible.\n";
576
0
0
0
0
            tear_here(3); return -1000;
577        }
578
1
50
6
        if ($gh_err_text =~ /HTTP 404: Not Found|"status":"404"/) {
579
1
16
            print "$program: The referenced repository ($repo) may not exist, perhaps you do not have permission to see it. If the repository is hosted by GitHub Enterprise, check-spelling does not know how to integrate with it.\n";
580
1
1
5
8
            tear_here(8); return -1000;
581        }
582
0
0
0
        if ($gh_err_text =~ /HTTP 403: API rate limit exceeded for .*?./) {
583        } elsif ($gh_err_text =~ m{dial tcp \S+:\d+: i/o timeout$}) {
584
0
0
            if ($retries_remaining <= 0) {
585
0
                print "$program: Timeout connecting to GitHub. This is probably caused by an outage of sorts.\nCheck https://www.githubstatus.com/history\nTry again later.";
586
0
0
                tear_here(9); return -1000;
587            }
588        } else {
589
0
            print "$program: Unknown error, please check the list of known issues https://github.com/check-spelling/check-spelling/issues?q=is%3Aissue%20apply.pl and file a bug to https://github.com/check-spelling/check-spelling/issues/new?title=%60apply.pl%60%20scenario&body=Please%20provide%20details+preferably%20including%20a%20link%20to%20a%20workflow%20run,%20the%20configuration%20of%20the%20repository,%20and%20anything%20else%20you%20may%20know%20about%20the%20problem%2e\n";
590
0
            print $gh_err_text;
591
0
0
            tear_here(4); return -1000;
592        }
593
0
0
        my $request_id = $1 if ($gh_err_text =~ /\brequest ID\s+(\S+)/);
594
0
0
        my $timestamp = $1 if ($gh_err_text =~ /\btimestamp\s+(.*? UTC)/);
595
0
0
        my $has_gh_token = defined $ENV{GH_TOKEN} || defined $ENV{GITHUB_TOKEN};
596
0
        my $meta_url = 'https://api.github.com/meta';
597
0
        while (1) {
598
0
            my @curl_args = qw(curl);
599
0
0
            unless ($has_gh_token) {
600
0
                my $gh_token = get_token();
601
0
0
                push @curl_args, '-u', "token:$gh_token" if defined $gh_token;
602            }
603
0
            push @curl_args, '-I', $meta_url;
604
0
            my ($curl_stdout, $curl_stderr, $curl_result);
605
0
            ($curl_stdout, $curl_stderr, $curl_result) = capture_system(@curl_args);
606
0
            my $delay = 1;
607
0
0
0
            if ($curl_stdout =~ m{^HTTP/\S+\s+200}) {
608
0
0
                if ($curl_stdout =~ m{^x-ratelimit-remaining:\s+(\d+)$}m) {
609
0
                    my $ratelimit_remaining = $1;
610
0
0
                    last if ($ratelimit_remaining > 10);
611
612
0
                    $delay = 5;
613
0
                    print STDERR "Sleeping for $delay seconds because $ratelimit_remaining is close to 0\n";
614                } else {
615
0
                    print STDERR "Couldn't find x-ratelimit-remaining, will sleep for $delay\n";
616                }
617            } elsif ($curl_stdout =~ m{^HTTP/\S+\s+403}) {
618
0
0
                if ($curl_stdout =~ /^retry-after:\s+(\d+)/m) {
619
0
                    $delay = $1;
620
0
                    print STDERR "Sleeping for $delay seconds (presumably due to API rate limit)\n";
621                } else {
622
0
                    print STDERR "Couldn't find retry-after, will sleep for $delay\n";
623                }
624            } else {
625
0
0
                my $response = $1 if $curl_stdout =~ m{^(HTTP/\S+)};
626
0
                print STDERR "Unexpected response ($response) from $meta_url; sleeping for $delay\n";
627            }
628
0
            sleep $delay;
629        }
630    }
631}
632
633sub update_repository {
634
0
0
    my ($artifact) = @_;
635
0
0
    die_custom 620, if $artifact =~ /'/;
636
0
    our $program;
637
0
    my $apply = unzip_pipe($artifact, 'apply.json');
638
0
0
    unless ($apply =~ /\{.*\}/s) {
639
0
        print STDERR "$program: Could not retrieve valid apply.json from artifact\n";
640
0
        $apply = '{
641            "expect_files": [".github/actions/spelling/expect.txt"],
642            "new_expect_file": ".github/actions/spelling/expect.txt",
643            "excludes_file": ".github/actions/spelling/excludes.txt",
644            "spelling_config": ".github/actions/spelling"
645        }';
646    }
647
0
    my $config_ref;
648
0
0
0
    eval { $config_ref = decode_json($apply); } ||
649        die_custom 634, "$program: decode_json failed in update_repository with '$apply'";
650
651
0
    my $git_repo_root = run_pipe('git', 'rev-parse', '--show-toplevel');
652
0
    chomp $git_repo_root;
653
0
0
    die_custom 638, "$program: Could not find git repo root..." unless $git_repo_root =~ /\w/;
654
0
    chdir $git_repo_root;
655
656
0
    retrieve_spell_check_this($artifact, $config_ref);
657
0
    remove_stale($artifact, $config_ref);
658
0
    add_expect($artifact, $config_ref);
659
0
    add_to_excludes($artifact, $config_ref);
660
0
    system('git', 'add', '-u', '--', $config_ref->{'spelling_config'});
661}
662
663sub extract_artifacts_from_file {
664
0
0
    my ($artifact) = @_;
665
0
    open my $artifact_reader, '-|', 'unzip', '-l', $artifact;
666
0
    my ($has_artifact, $only_file) = (0, 0);
667
0
    while (my $line = <$artifact_reader>) {
668
0
        chomp $line;
669
0
0
        if ($line =~ /\s+artifact\.zip$/) {
670
0
            $has_artifact = 1;
671
0
            next;
672        }
673
0
0
        if ($line =~ /\s+1 file$/) {
674
0
            $only_file = 1;
675
0
            next;
676        }
677
0
0
        $only_file = 0 if $only_file;
678    }
679
0
    close $artifact_reader;
680
0
    my @artifacts;
681
0
0
0
    if ($has_artifact && $only_file) {
682
0
        my $artifact_dir = tempdir(CLEANUP => 1);
683
0
        my ($fh, $gh_err) = tempfile();
684
0
        close $fh;
685
0
        system('unzip', '-q', '-d', $artifact_dir, $artifact, 'artifact.zip');
686
0
        @artifacts = ("$artifact_dir/artifact.zip");
687    } else {
688
0
        @artifacts = ($artifact);
689    }
690
0
    return @artifacts;
691}
692
693sub main {
694
0
0
    our $program;
695
0
    my ($bash_script, $first, $run);
696
0
    ($program, $bash_script, $first, $run) = @_;
697
0
    my $syntax = "$program <RUN_URL | OWNER/REPO RUN | ARTIFACT.zip>";
698    # Stages
699    # - 1 check for tools basic
700
0
    check_basic_tools();
701    # - 2 check for current
702    # -> 1. download the latest version to a temp file
703    # -> 2. parse current and latest (stripping comments) and compare (whitespace insensitively)
704    # -> 3. offer to update if the latest version is different
705
0
    check_current_script($bash_script);
706    # - 4 parse arguments
707
0
0
    die_custom 692, $syntax unless defined $first;
708
0
0
    $ENV{'GITHUB_API_URL'} ||= 'https://api.github.com';
709
0
    my $repo;
710    my @artifacts;
711
0
0
    if (-s $first) {
712
0
        @artifacts = extract_artifacts_from_file($first);
713    } else {
714
0
        my $suffix;
715
0
0
        if ($first =~ m{^\s*https://.*/([^/]+/[^/]+)/actions/runs/(\d+)(?:/attempts/\d+|)(?:#(\S+)|)\s*$}) {
716
0
            ($repo, $run, $suffix) = ($1, $2, $3);
717        } else {
718
0
            $repo = $first;
719        }
720
0
0
0
        die_custom 705, $syntax unless defined $repo && defined $run;
721        # - 3 check for tool readiness (is `gh` working)
722
0
        tools_are_ready($program);
723
0
        @artifacts = get_artifacts($repo, $run, $suffix);
724    }
725
726    # - 5 do work
727
0
    for my $artifact (@artifacts) {
728
0
        update_repository($artifact);
729    }
730}
731