xref: /curl/scripts/cijobs.pl (revision 922091c0) 
1#!/usr/bin/perl
2#***************************************************************************
3#                                  _   _ ____  _
4#  Project                     ___| | | |  _ \| |
5#                             / __| | | | |_) | |
6#                            | (__| |_| |  _ <| |___
7#                             \___|\___/|_| \_\_____|
8#
9# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
10#
11# This software is licensed as described in the file COPYING, which
12# you should have received as part of this distribution. The terms
13# are also available at https://curl.se/docs/copyright.html.
14#
15# You may opt to use, copy, modify, merge, publish, distribute and/or sell
16# copies of the Software, and permit persons to whom the Software is
17# furnished to do so, under the terms of the COPYING file.
18#
19# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
20# KIND, either express or implied.
21#
22# SPDX-License-Identifier: curl
23#
24###########################################################################
25
26my %filelevel= ('file' => 1,
27                'service' => 1);
28
29my $jobid = 1;
30
31sub submit {
32    my ($jref)=@_;
33    my %job = %$jref;
34
35    printf "\n##### job %u \n", $jobid++;
36    for my $k (sort keys %job) {
37        printf "%s: %s\n", $k, $job{$k} if($job{$k});
38        undef $$jref{$k} if(!$filelevel{$k});
39    }
40}
41
42sub githubactions {
43    my ($tag)=@_;
44    my @files= `git ls-tree -r --name-only $tag .github/workflows 2>/dev/null`;
45    my $c = 0;
46    foreach my $f (sort @files) {
47        my $j = 0;
48        my $m = -1;
49        my $done = 0;
50        chomp $f;
51        open(G, "git show $tag:$f 2>/dev/null|");
52        # start counting file jobs
53        undef %job;
54        $job{'file'} = $f;
55        $job{'service'} = "gha";
56        my @cc;
57        my $os;
58        my $topname;
59        my $line = 1;
60        while(<G>) {
61            $job{'line'} = $line;
62            if($_ =~ /^name: (.*)/) {
63                $topname=$1;
64            }
65            elsif($_ =~ /runs-on: (.*)/) {
66                my $r = $1;
67                #print "runs-on: $r\n";
68                if($r =~ /ubuntu/) {
69                    $os = "linux";
70                }
71                elsif($r =~ /macos/) {
72                    $os = "macos";
73                }
74                elsif($r =~ /windows/) {
75                    $os = "windows";
76                }
77
78                # commit previously counted jobs
79                $c += $j;
80                # non-matrix job
81                $j = 1;
82            }
83            elsif($_ =~ /^\s*matrix:/) {
84                # switch to matrix mode
85                $m = 0;
86                $j = 0;
87            }
88            elsif($_ =~ /^    - run: .* apt-get install (.*)/) {
89                $job{'install'} = $1;
90            }
91            elsif($m >= 0) {
92                if($_ =~ /^          - name: (.*)/) {
93                    # matrix job
94                    #print "name: $1\n";
95                    $job{'name'} = $1;
96                    $j += ($m?$m:1);
97                }
98                elsif($_ =~ /install: (.*)/) {
99                    $job{'install'} = $1;
100                }
101                elsif($_ =~ /( |curl-)configure: (.*)/) {
102                    $job{'configure'} = $2;
103                    $job{'os'}=$os;
104                    submit(\%job);
105                    $done++;
106                }
107                elsif($_ =~ /generate: (.*)/) {
108                    $job{'cmake'} = $1;
109                    if($m) {
110                        # matrix mode, multiple copies
111                        my %dupe = %job;
112                        for my $cc (@cc) {
113                            %job = %dupe;
114                            $job{'cc'} = $cc;
115                            $job{'os'}=$os;
116                            submit(\%job);
117                            $done++;
118                        }
119                    }
120                    else {
121                        $job{'os'}=$os;
122                        submit(\%job);
123                        $done++;
124                    }
125                }
126                elsif($_ =~ /- CC: (.*)/) {
127                    # matrix multiplier
128                    push @cc, $1;
129                    $m++;
130                }
131                elsif($_ =~ /^\s*steps:/) {
132                    # disable matrix mode
133                    $m = -1;
134                }
135            }
136            $line++;
137        }
138        close(G);
139        # commit final counted jobs
140        $c += $j;
141
142        if(!$done) {
143            $job{'name'} = $topname? $topname : '[unnamed]';
144            $job{'os'}=$os;
145            submit(\%job);
146            $done++;
147        }
148        # reset internal job counter
149        $j = 0;
150    }
151    #print "Jobs: $c\n";
152    return $c;
153}
154
155sub azurepipelines {
156    my ($tag)=@_;
157    open(G, "git show $tag:.azure-pipelines.yml 2>/dev/null|");
158    my $c = 0;
159    my $j = 0;
160    my $m = -1;
161    my $image;
162    my %job;
163    my $line = 1;
164    my $os;
165    $job{'file'} = ".azure-pipelines.yml";
166    $job{'service'} = "azure";
167    while(<G>) {
168        if($_ =~ /^          vmImage: (.*)/) {
169            my $i = $1;
170            if($i =~ /ubuntu/) {
171                $os = "linux";
172            }
173            elsif($i =~ /windows/) {
174                $os = "windows";
175            }
176        }
177        elsif($_ =~ /^  - stage: (.*)/) {
178            my $topname = $1;
179            if($topname !~ /(windows|linux)/) {
180                $job{'name'} = $topname;
181                $job{'line'}=$line;
182                submit(\%job);
183            }
184        }
185        elsif($_ =~ /job:/) {
186            # commit previously counted jobs
187            $c += $j;
188            # initial value for non-matrix job
189            $j = 1;
190        }
191        elsif($_ =~ /matrix:/) {
192            # start of new matrix list(!)
193            $m = 0;
194            $j = 0;
195        }
196        elsif($m >= 0) {
197            if($_ =~ /^              name: (.*)/) {
198                # single matrix list entry job
199                $j++;
200                $job{'name'} = $1;
201            }
202            # azure matrix is a simple list,
203            # therefore no multiplier needed
204            elsif($_ =~ /steps:/) {
205                # disable matrix mode
206                $m = -1;
207            }
208            elsif($_ =~ /^              configure: (.*)/) {
209                $job{'configure'} = $1;
210                $job{'line'}=$line;
211                $job{'os'}=$os;
212                submit(\%job);
213            }
214        }
215        $line++;
216    }
217    close(G);
218    # commit final counted jobs
219    $c += $j;
220
221    return $c;
222}
223
224sub appveyor {
225    my ($tag)=@_;
226    open(G, "git show $tag:appveyor.yml 2>/dev/null|");
227    my $c = 0;
228    my %job;
229    my $line=0;
230    $job{'file'} = "appveyor.yml";
231    $job{'service'} = "appveyor";
232
233    while(<G>) {
234        $line++;
235        if($_ =~ /^(    - |install)/) {
236            if($job{'image'}) {
237                $job{'os'} = "windows";
238                submit(\%job);
239                $c++;
240            }
241        }
242        $job{'line'} = $line;
243        if($_ =~ /^      APPVEYOR_BUILD_WORKER_IMAGE: \'(.*)\'/) {
244            $job{'image'}= $1;
245        }
246        elsif($_ =~ /^      BUILD_SYSTEM: (.*)/) {
247            $job{'build'} = lc($1);
248        }
249        elsif($_ =~ /^      PRJ_GEN: \'(.*)\'/) {
250            $job{'compiler'} = $1;
251        }
252        elsif($_ =~ /^      PRJ_CFG: (.*)/) {
253            $job{'config'} = $1;
254        }
255        elsif($_ =~ /^      OPENSSL: \'(.*)\'/) {
256            $job{'openssl'} = $1 eq "ON" ? "true": "false";
257        }
258        elsif($_ =~ /^      SCHANNEL: \'(.*)\'/) {
259            $job{'schannel'} = $1 eq "ON" ? "true": "false";
260        }
261        elsif($_ =~ /^      ENABLE_UNICODE: \'(.*)\'/) {
262            $job{'unicode'} = $1 eq "ON" ? "true": "false";
263        }
264        elsif($_ =~ /^      HTTP_ONLY: \'(.*)\'/) {
265            $job{'http-only'} = $1 eq "ON" ? "true": "false";
266        }
267        elsif($_ =~ /^      TESTING: \'(.*)\'/) {
268            $job{'testing'} = $1 eq "ON" ? "true": "false";
269        }
270        elsif($_ =~ /^      SHARED: \'(.*)\'/) {
271            $job{'shared'} = $1 eq "ON" ? "true": "false";
272        }
273        elsif($_ =~ /^      TARGET: \'-A (.*)\'/) {
274            $job{'target'} = $1;
275        }
276    }
277    close(G);
278
279    return $c;
280}
281
282sub cirrus {
283    my ($tag)=@_;
284    open(G, "git show $tag:.cirrus.yml 2>/dev/null|");
285    my $c = 0;
286    my %job;
287    my $line=0;
288    my $name = 0;
289    my $os;
290    $job{'file'} = ".cirrus.yml";
291    $job{'service'} = "cirrus";
292    while(<G>) {
293        $line++;
294        if($_ =~ /^    ( |-) (name|image_family|image):/) {
295            $c++;
296        }
297        if($_ =~ /^    - name:/) {
298            if($name) {
299                $job{'os'} = $os;
300                $job{'line'} = $line;
301                submit(\%job);
302                $name = 0;
303            }
304        }
305        if($_ =~ /^    - name: (.*)/) {
306            $job{'name'} = $1;
307            $name = 1;
308        }
309        elsif($_ =~ /^        image_family: (.*)/) {
310            $os = "freebsd";
311        }
312        elsif($_ =~ /^windows_task:/) {
313            $os = "windows";
314        }
315        elsif($_ =~ /^        prepare: pacman -S --needed --noconfirm --noprogressbar (.*)/) {
316            $job{'install'} = $1;
317        }
318        elsif($_ =~ /^        configure: (.*)/) {
319            $job{'configure'} = $1;
320        }
321    }
322    close(G);
323    if($name) {
324        $job{'os'} = $os;
325        $job{'line'} = $line;
326        submit(\%job);
327    }
328    return $c;
329}
330
331sub circle {
332    my ($tag)=@_;
333    open(G, "git show $tag:.circleci/config.yml 2>/dev/null|");
334    my $c = 0;
335    my $wf = 0;
336    my %job;
337    my %cmd;
338    my %configure;
339    my %target;
340    my $line=0;
341    my $cmds;
342    my $jobs;
343    my $workflow;
344    $job{'file'} = ".circleci/config.yml";
345    $job{'service'} = "circleci";
346    while(<G>) {
347        $line++;
348        if($_ =~ /^commands:/) {
349            # we record configure lines in this state
350            $cmds = 1;
351        }
352        elsif($cmds) {
353            if($_ =~ /^  ([^ ]*):/) {
354                $cmdname = $1;
355            }
356            elsif($_ =~ /^            .*.\/configure (.*)/) {
357                $cmd{$cmdname}=$1;
358            }
359        }
360        if($_ =~ /^jobs:/) {
361            # we record which job runs with configure here
362            $jobs = 1;
363            $cmds = 0;
364        }
365        elsif($jobs) {
366            if($_ =~ /^  ([^ ]*):/) {
367                $jobname = $1;
368            }
369            elsif($_ =~ /^      - (configure.*)/) {
370                $configure{$jobname}=$1;
371            }
372            elsif($_ =~ /^    resource_class: arm.medium/) {
373                $target{$jobname}="arm";
374            }
375        }
376        if($_ =~ /^workflows:/) {
377            $wf = 1;
378            $cmds = 0;
379        }
380        elsif($wf) {
381            if($_ =~ /^  ([^ ]+):/) {
382                $workflow = $1;
383            }
384            elsif($_ =~ /^      - (.*)\n/) {
385                my $jb = $1;
386                my $cnfgure = $configure{$jb};
387                my $trgt = $target{$jb};
388                $job{'configure'} = $cmd{$cnfgure};
389                $job{'name' }=$workflow;
390                $job{'os'} = "linux";
391                $job{'line'} = $line;
392                $job{'target'} = $trgt if($trgt);
393                submit(\%job);
394            }
395            if($_ =~ / *jobs:/) {
396                $c++;
397            }
398        }
399    }
400    close(G);
401    return $c;
402}
403
404sub zuul {
405    my ($tag)=@_;
406    open(G, "git show $tag:zuul.d/jobs.yaml 2>/dev/null|");
407    my $c = 0;
408    my %job;
409    my $line=0;
410    my $type;
411    $job{'file'} = "zuul.d/jobs.yaml";
412    $job{'service'} = "zuul";
413    while(<G>) {
414        $line++;
415        #print "L: ($jobmode / $env) $_";
416        if($_ =~ /^- job:/) {
417            $jobmode = 1; # start a new
418            $type="configure";
419        }
420        if($jobmode) {
421            if($apt) {
422                if($_ =~ /^        - (.*)/) {
423                    my $value = $1;
424                    $job{'install'} .= "$value ";
425                }
426                else {
427                    $apt = 0; # end of curl_apt_packages
428                }
429            }
430            if($env) {
431                if($envcont) {
432                    if($_ =~ /^          (.*)/) {
433                        $job{$envcont} .= "$1 ";
434                    }
435                    else {
436                        $envcont = "";
437                    }
438                }
439                if($_ =~ /^        ([^:]+): (.*)/) {
440                    my ($var, $value) = ($1, $2);
441
442                    if($var eq "C") {
443                        $var = $type;
444                    }
445                    elsif($var eq "T") {
446                        $var = "tests";
447                        if($value eq "cmake") {
448                            # otherwise it remains configure
449                            $type = "cmake";
450                        }
451                    }
452                    elsif($var eq "CC") {
453                        $var = "compiler";
454                    }
455                    elsif($var eq "CHECKSRC") {
456                        $job{'checksrc'} = $value ? "true": "false";
457                        $var = "";
458                    }
459                    else {
460                        $var = "";
461                    }
462                    if($value eq ">-") {
463                        $envcont = $var;
464                    }
465                    elsif($var) {
466                        $job{$var} = $value;
467                    }
468                }
469                elsif($_ !~ /^        /) {
470                    # end of envs
471                    $env = 0;
472                }
473            }
474            if($_ =~ /^      curl_env:/) {
475                $env = 1; # start of envs
476            }
477            elsif($_ =~ /^      curl_apt_packages:/) {
478                $apt = 1; # start of apt packages
479            }
480            elsif($_ =~ /^    name: (.*)/) {
481                my $n = $1;
482                if($n eq "curl-base") {
483                    # not counted
484                    $jobmode = 0;
485                    next;
486                }
487                $job{'name'} = $n;
488            }
489            elsif($_ =~ /^\n\z/) {
490                # a job is complete
491                $job{'line'}=$line;
492                $job{'os'}="linux";
493                submit(\%job);
494                $jobmode = 0;
495                $c++;
496            }
497        }
498    }
499    close(G);
500    return $c;
501}
502
503my $tag = `git rev-parse --abbrev-ref HEAD 2>/dev/null` || "master";
504chomp $tag;
505githubactions($tag);
506azurepipelines($tag);
507appveyor($tag);
508zuul($tag);
509cirrus($tag);
510circle($tag);
511