Print this page
4445 wsdiff results file timestamp format is strange
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/tools/scripts/wsdiff.py
+++ new/usr/src/tools/scripts/wsdiff.py
1 1 #!/usr/bin/python2.6
2 2 #
3 3 # CDDL HEADER START
4 4 #
5 5 # The contents of this file are subject to the terms of the
6 6 # Common Development and Distribution License (the "License").
7 7 # You may not use this file except in compliance with the License.
8 8 #
9 9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 10 # or http://www.opensolaris.org/os/licensing.
11 11 # See the License for the specific language governing permissions
12 12 # and limitations under the License.
13 13 #
14 14 # When distributing Covered Code, include this CDDL HEADER in each
15 15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 16 # If applicable, add the following below this CDDL HEADER, with the
17 17 # fields enclosed by brackets "[]" replaced with your own identifying
18 18 # information: Portions Copyright [yyyy] [name of copyright owner]
19 19 #
20 20 # CDDL HEADER END
21 21 #
22 22 # Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
23 23 #
24 24
25 25 #
26 26 # wsdiff(1) is a tool that can be used to determine which compiled objects
27 27 # have changed as a result of a given source change. Developers backporting
28 28 # new features, RFEs and bug fixes need to be able to identify the set of
29 29 # patch deliverables necessary for feature/fix realization on a patched system.
30 30 #
31 31 # The tool works by comparing objects in two trees/proto areas (one build with,
32 32 # and without the source changes.
33 33 #
34 34 # Using wsdiff(1) is fairly simple:
35 35 # - Bringover to a fresh workspace
36 36 # - Perform a full non-debug build (clobber if workspace isn't fresh)
37 37 # - Move the proto area aside, call it proto.old, or something.
38 38 # - Integrate your changes to the workspace
39 39 # - Perform another full non-debug clobber build.
40 40 # - Use wsdiff(1) to see what changed:
41 41 # $ wsdiff proto.old proto
42 42 #
43 43 # By default, wsdiff will print the list of changed objects / deliverables to
44 44 # stdout. If a results file is specified via -r, the list of differing objects,
45 45 # and details about why wsdiff(1) thinks they are different will be logged to
46 46 # the results file.
47 47 #
48 48 # By invoking nightly(1) with the -w option to NIGHTLY_FLAGS, nightly(1) will use
49 49 # wsdiff(1) to report on what objects changed since the last build.
50 50 #
51 51 # For patch deliverable purposes, it's advised to have nightly do a clobber,
52 52 # non-debug build.
53 53 #
54 54 # Think about the results. Was something flagged that you don't expect? Go look
55 55 # at the results file to see details about the differences.
56 56 #
57 57 # Use the -i option in conjunction with -v and -V to dive deeper and have wsdiff(1)
58 58 # report with more verbosity.
59 59 #
60 60 # Usage: wsdiff [-vVt] [-r results ] [-i filelist ] old new
61 61 #
62 62 # Where "old" is the path to the proto area build without the changes, and
63 63 # "new" is the path to the proto area built with the changes. The following
64 64 # options are supported:
65 65 #
66 66 # -v Do not truncate observed diffs in results
67 67 # -V Log *all* ELF sect diffs vs. logging the first diff found
68 68 # -t Use onbld tools in $SRC/tools
69 69 # -r Log results and observed differences
70 70 # -i Tell wsdiff which objects to compare via an input file list
71 71
72 72 import datetime, fnmatch, getopt, os, profile, commands
73 73 import re, resource, select, shutil, signal, string, struct, sys, tempfile
74 74 import time, threading
75 75 from stat import *
76 76
77 77 # Human readable diffs truncated by default if longer than this
78 78 # Specifying -v on the command line will override
79 79 diffs_sz_thresh = 4096
80 80
81 81 # Lock name Provides exclusive access to
82 82 # --------------+------------------------------------------------
83 83 # output_lock standard output or temporary file (difference())
84 84 # log_lock the results file (log_difference())
85 85 # wset_lock changedFiles list (workerThread())
86 86 output_lock = threading.Lock()
87 87 log_lock = threading.Lock()
88 88 wset_lock = threading.Lock()
89 89
90 90 # Variable for thread control
91 91 keep_processing = True
92 92
93 93 # Default search path for wsdiff
94 94 wsdiff_path = [ "/usr/bin",
95 95 "/usr/ccs/bin",
96 96 "/lib/svc/bin",
97 97 "/opt/onbld/bin" ]
98 98
99 99 # These are objects that wsdiff will notice look different, but will not report.
100 100 # Existence of an exceptions list, and adding things here is *dangerous*,
101 101 # and therefore the *only* reasons why anything would be listed here is because
102 102 # the objects do not build deterministically, yet we *cannot* fix this.
103 103 #
104 104 # These perl libraries use __DATE__ and therefore always look different.
105 105 # Ideally, we would purge use the use of __DATE__ from the source, but because
106 106 # this is source we wish to distribute with Solaris "unchanged", we cannot modify.
107 107 #
108 108 wsdiff_exceptions = [ "usr/perl5/5.8.4/lib/sun4-solaris-64int/CORE/libperl.so.1",
109 109 "usr/perl5/5.6.1/lib/sun4-solaris-64int/CORE/libperl.so.1",
110 110 "usr/perl5/5.8.4/lib/i86pc-solaris-64int/CORE/libperl.so.1",
111 111 "usr/perl5/5.6.1/lib/i86pc-solaris-64int/CORE/libperl.so.1"
112 112 ]
113 113
114 114 #####
115 115 # Logging routines
116 116 #
117 117
118 118 # Debug message to be printed to the screen, and the log file
119 119 def debug(msg) :
120 120
121 121 # Add prefix to highlight debugging message
122 122 msg = "## " + msg
123 123 if debugon :
124 124 output_lock.acquire()
125 125 print >> sys.stdout, msg
126 126 sys.stdout.flush()
127 127 output_lock.release()
128 128 if logging :
129 129 log_lock.acquire()
130 130 print >> log, msg
131 131 log.flush()
132 132 log_lock.release()
133 133
134 134 # Informational message to be printed to the screen, and the log file
135 135 def info(msg) :
136 136
137 137 output_lock.acquire()
138 138 print >> sys.stdout, msg
139 139 sys.stdout.flush()
140 140 output_lock.release()
141 141 if logging :
142 142 log_lock.acquire()
143 143 print >> log, msg
144 144 log.flush()
145 145 log_lock.release()
146 146
147 147 # Error message to be printed to the screen, and the log file
148 148 def error(msg) :
149 149
150 150 output_lock.acquire()
151 151 print >> sys.stderr, "ERROR:", msg
152 152 sys.stderr.flush()
153 153 output_lock.release()
154 154 if logging :
155 155 log_lock.acquire()
156 156 print >> log, "ERROR:", msg
157 157 log.flush()
158 158 log_lock.release()
159 159
160 160 # Informational message to be printed only to the log, if there is one.
161 161 def v_info(msg) :
162 162
163 163 if logging :
164 164 log_lock.acquire()
165 165 print >> log, msg
166 166 log.flush()
167 167 log_lock.release()
168 168
169 169 #
170 170 # Flag a detected file difference
171 171 # Display the fileName to stdout, and log the difference
172 172 #
173 173 def difference(f, dtype, diffs) :
174 174
175 175 if f in wsdiff_exceptions :
176 176 return
177 177
178 178 output_lock.acquire()
179 179 if sorted :
180 180 differentFiles.append(f)
181 181 else:
182 182 print >> sys.stdout, f
183 183 sys.stdout.flush()
184 184 output_lock.release()
185 185
186 186 log_difference(f, dtype, diffs)
187 187
188 188 #
189 189 # Do the actual logging of the difference to the results file
190 190 #
191 191 def log_difference(f, dtype, diffs) :
192 192
193 193 if logging :
194 194 log_lock.acquire()
195 195 print >> log, f
196 196 print >> log, "NOTE:", dtype, "difference detected."
197 197
198 198 difflen = len(diffs)
199 199 if difflen > 0 :
200 200 print >> log
201 201
202 202 if not vdiffs and difflen > diffs_sz_thresh :
203 203 print >> log, diffs[:diffs_sz_thresh]
204 204 print >> log, \
205 205 "... truncated due to length: " \
206 206 "use -v to override ..."
207 207 else :
208 208 print >> log, diffs
209 209 print >> log, "\n"
210 210 log.flush()
211 211 log_lock.release()
212 212
213 213
214 214 #####
215 215 # diff generating routines
216 216 #
217 217
218 218 #
219 219 # Return human readable diffs from two temporary files
220 220 #
221 221 def diffFileData(tmpf1, tmpf2) :
222 222
223 223 binaries = False
224 224
225 225 # Filter the data through od(1) if the data is detected
226 226 # as being binary
227 227 if isBinary(tmpf1) or isBinary(tmpf2) :
228 228 binaries = True
229 229 tmp_od1 = tmpf1 + ".od"
230 230 tmp_od2 = tmpf2 + ".od"
231 231
232 232 cmd = od_cmd + " -c -t x4" + " " + tmpf1 + " > " + tmp_od1
233 233 os.system(cmd)
234 234 cmd = od_cmd + " -c -t x4" + " " + tmpf2 + " > " + tmp_od2
235 235 os.system(cmd)
236 236
237 237 tmpf1 = tmp_od1
238 238 tmpf2 = tmp_od2
239 239
240 240 try:
241 241 data = commands.getoutput(diff_cmd + " " + tmpf1 + " " + tmpf2)
242 242 # Remove the temp files as we no longer need them.
243 243 if binaries :
244 244 try:
245 245 os.unlink(tmp_od1)
246 246 except OSError, e:
247 247 error("diffFileData: unlink failed %s" % e)
248 248 try:
249 249 os.unlink(tmp_od2)
250 250 except OSError, e:
251 251 error("diffFileData: unlink failed %s" % e)
252 252 except:
253 253 error("failed to get output of command: " + diff_cmd + " " \
254 254 + tmpf1 + " " + tmpf2)
255 255
256 256 # Send exception for the failed command up
257 257 raise
258 258 return
259 259
260 260 return data
261 261
262 262 #
263 263 # Return human readable diffs betweeen two datasets
264 264 #
265 265 def diffData(base, ptch, d1, d2) :
266 266
267 267 t = threading.currentThread()
268 268 tmpFile1 = tmpDir1 + os.path.basename(base) + t.getName()
269 269 tmpFile2 = tmpDir2 + os.path.basename(ptch) + t.getName()
270 270
271 271 try:
272 272 fd1 = open(tmpFile1, "w")
273 273 except:
274 274 error("failed to open: " + tmpFile1)
275 275 cleanup(1)
276 276
277 277 try:
278 278 fd2 = open(tmpFile2, "w")
279 279 except:
280 280 error("failed to open: " + tmpFile2)
281 281 cleanup(1)
282 282
283 283 fd1.write(d1)
284 284 fd2.write(d2)
285 285 fd1.close()
286 286 fd2.close()
287 287
288 288 return diffFileData(tmpFile1, tmpFile2)
289 289
290 290 #####
291 291 # Misc utility functions
292 292 #
293 293
294 294 # Prune off the leading prefix from string s
295 295 def str_prefix_trunc(s, prefix) :
296 296 snipLen = len(prefix)
297 297 return s[snipLen:]
298 298
299 299 #
300 300 # Prune off leading proto path goo (if there is one) to yield
301 301 # the deliverable's eventual path relative to root
302 302 # e.g. proto.base/root_sparc/usr/src/cmd/prstat => usr/src/cmd/prstat
303 303 #
304 304 def fnFormat(fn) :
305 305 root_arch_str = "root_" + arch
306 306
307 307 pos = fn.find(root_arch_str)
308 308 if pos == -1 :
309 309 return fn
310 310
311 311 pos = fn.find("/", pos)
312 312 if pos == -1 :
313 313 return fn
314 314
315 315 return fn[pos + 1:]
316 316
317 317 #####
318 318 # Usage / argument processing
319 319 #
320 320
321 321 #
322 322 # Display usage message
323 323 #
324 324 def usage() :
325 325 sys.stdout.flush()
326 326 print >> sys.stderr, """Usage: wsdiff [-dvVst] [-r results ] [-i filelist ] old new
327 327 -d Print debug messages about the progress
328 328 -v Do not truncate observed diffs in results
329 329 -V Log *all* ELF sect diffs vs. logging the first diff found
330 330 -t Use onbld tools in $SRC/tools
331 331 -r Log results and observed differences
332 332 -s Produce sorted list of differences
333 333 -i Tell wsdiff which objects to compare via an input file list"""
334 334 sys.exit(1)
335 335
336 336 #
337 337 # Process command line options
338 338 #
339 339 def args() :
340 340
341 341 global debugon
342 342 global logging
343 343 global vdiffs
344 344 global reportAllSects
345 345 global sorted
346 346
347 347 validOpts = 'di:r:vVst?'
348 348
349 349 baseRoot = ""
350 350 ptchRoot = ""
351 351 fileNamesFile = ""
352 352 results = ""
353 353 localTools = False
354 354
355 355 # getopt.getopt() returns:
356 356 # an option/value tuple
357 357 # a list of remaining non-option arguments
358 358 #
359 359 # A correct wsdiff invocation will have exactly two non option
360 360 # arguments, the paths to the base (old), ptch (new) proto areas
361 361 try:
362 362 optlist, args = getopt.getopt(sys.argv[1:], validOpts)
363 363 except getopt.error, val:
364 364 usage()
365 365
366 366 if len(args) != 2 :
367 367 usage();
368 368
369 369 for opt,val in optlist :
370 370 if opt == '-d' :
371 371 debugon = True
372 372 elif opt == '-i' :
373 373 fileNamesFile = val
374 374 elif opt == '-r' :
375 375 results = val
376 376 logging = True
377 377 elif opt == '-s' :
378 378 sorted = True
379 379 elif opt == '-v' :
380 380 vdiffs = True
381 381 elif opt == '-V' :
382 382 reportAllSects = True
383 383 elif opt == '-t':
384 384 localTools = True
385 385 else:
386 386 usage()
387 387
388 388 baseRoot = args[0]
389 389 ptchRoot = args[1]
390 390
391 391 if len(baseRoot) == 0 or len(ptchRoot) == 0 :
392 392 usage()
393 393
394 394 if logging and len(results) == 0 :
395 395 usage()
396 396
397 397 if vdiffs and not logging :
398 398 error("The -v option requires a results file (-r)")
399 399 sys.exit(1)
400 400
401 401 if reportAllSects and not logging :
402 402 error("The -V option requires a results file (-r)")
403 403 sys.exit(1)
404 404
405 405 # alphabetical order
406 406 return baseRoot, fileNamesFile, localTools, ptchRoot, results
407 407
408 408 #####
409 409 # File identification
410 410 #
411 411
412 412 #
413 413 # Identify the file type.
414 414 # If it's not ELF, use the file extension to identify
415 415 # certain file types that require special handling to
416 416 # compare. Otherwise just return a basic "ASCII" type.
417 417 #
418 418 def getTheFileType(f) :
419 419
420 420 extensions = { 'a' : 'ELF Object Archive',
421 421 'jar' : 'Java Archive',
422 422 'html' : 'HTML',
423 423 'ln' : 'Lint Library',
424 424 'db' : 'Sqlite Database' }
425 425
426 426 try:
427 427 if os.stat(f)[ST_SIZE] == 0 :
428 428 return 'ASCII'
429 429 except:
430 430 error("failed to stat " + f)
431 431 return 'Error'
432 432
433 433 if isELF(f) == 1 :
434 434 return 'ELF'
435 435
436 436 fnamelist = f.split('.')
437 437 if len(fnamelist) > 1 : # Test the file extension
438 438 extension = fnamelist[-1]
439 439 if extension in extensions.keys():
440 440 return extensions[extension]
441 441
442 442 return 'ASCII'
443 443
444 444 #
445 445 # Return non-zero if "f" is an ELF file
446 446 #
447 447 elfmagic = '\177ELF'
448 448 def isELF(f) :
449 449 try:
450 450 fd = open(f)
451 451 except:
452 452 error("failed to open: " + f)
453 453 return 0
454 454 magic = fd.read(len(elfmagic))
455 455 fd.close()
456 456
457 457 if magic == elfmagic :
458 458 return 1
459 459 return 0
460 460
461 461 #
462 462 # Return non-zero is "f" is binary.
463 463 # Consider the file to be binary if it contains any null characters
464 464 #
465 465 def isBinary(f) :
466 466 try:
467 467 fd = open(f)
468 468 except:
469 469 error("failed to open: " + f)
470 470 return 0
471 471 s = fd.read()
472 472 fd.close()
473 473
474 474 if s.find('\0') == -1 :
475 475 return 0
476 476 else :
477 477 return 1
478 478
479 479 #####
480 480 # Directory traversal and file finding
481 481 #
482 482
483 483 #
484 484 # Return a sorted list of files found under the specified directory
485 485 #
486 486 def findFiles(d) :
487 487 for path, subdirs, files in os.walk(d) :
488 488 files.sort()
489 489 for name in files :
490 490 yield os.path.join(path, name)
491 491
492 492 #
493 493 # Examine all files in base, ptch
494 494 #
495 495 # Return a list of files appearing in both proto areas,
496 496 # a list of new files (files found only in ptch) and
497 497 # a list of deleted files (files found only in base)
498 498 #
499 499 def protoCatalog(base, ptch) :
500 500
501 501 compFiles = [] # List of files in both proto areas
502 502 ptchList = [] # List of file in patch proto area
503 503
504 504 newFiles = [] # New files detected
505 505 deletedFiles = [] # Deleted files
506 506
507 507 debug("Getting the list of files in the base area");
508 508 baseFilesList = list(findFiles(base))
509 509 baseStringLength = len(base)
510 510 debug("Found " + str(len(baseFilesList)) + " files")
511 511
512 512 debug("Getting the list of files in the patch area");
513 513 ptchFilesList = list(findFiles(ptch))
514 514 ptchStringLength = len(ptch)
515 515 debug("Found " + str(len(ptchFilesList)) + " files")
516 516
517 517 # Inventory files in the base proto area
518 518 debug("Determining the list of regular files in the base area");
519 519 for fn in baseFilesList :
520 520 if os.path.islink(fn) :
521 521 continue
522 522
523 523 fileName = fn[baseStringLength:]
524 524 compFiles.append(fileName)
525 525 debug("Found " + str(len(compFiles)) + " files")
526 526
527 527 # Inventory files in the patch proto area
528 528 debug("Determining the list of regular files in the patch area");
529 529 for fn in ptchFilesList :
530 530 if os.path.islink(fn) :
531 531 continue
532 532
533 533 fileName = fn[ptchStringLength:]
534 534 ptchList.append(fileName)
535 535 debug("Found " + str(len(ptchList)) + " files")
536 536
537 537 # Deleted files appear in the base area, but not the patch area
538 538 debug("Searching for deleted files by comparing the lists")
539 539 for fileName in compFiles :
540 540 if not fileName in ptchList :
541 541 deletedFiles.append(fileName)
542 542 debug("Found " + str(len(deletedFiles)) + " deleted files")
543 543
544 544 # Eliminate "deleted" files from the list of objects appearing
545 545 # in both the base and patch proto areas
546 546 debug("Eliminating deleted files from the list of objects")
547 547 for fileName in deletedFiles :
548 548 try:
549 549 compFiles.remove(fileName)
550 550 except:
551 551 error("filelist.remove() failed")
552 552 debug("List for comparison reduced to " + str(len(compFiles)) \
553 553 + " files")
554 554
555 555 # New files appear in the patch area, but not the base
556 556 debug("Getting the list of newly added files")
557 557 for fileName in ptchList :
558 558 if not fileName in compFiles :
559 559 newFiles.append(fileName)
560 560 debug("Found " + str(len(newFiles)) + " new files")
561 561
562 562 return compFiles, newFiles, deletedFiles
563 563
564 564 #
565 565 # Examine the files listed in the input file list
566 566 #
567 567 # Return a list of files appearing in both proto areas,
568 568 # a list of new files (files found only in ptch) and
569 569 # a list of deleted files (files found only in base)
570 570 #
571 571 def flistCatalog(base, ptch, flist) :
572 572 compFiles = [] # List of files in both proto areas
573 573 newFiles = [] # New files detected
574 574 deletedFiles = [] # Deleted files
575 575
576 576 try:
577 577 fd = open(flist, "r")
578 578 except:
579 579 error("could not open: " + flist)
580 580 cleanup(1)
581 581
582 582 files = []
583 583 files = fd.readlines()
584 584 fd.close()
585 585
586 586 for f in files :
587 587 ptch_present = True
588 588 base_present = True
589 589
590 590 if f == '\n' :
591 591 continue
592 592
593 593 # the fileNames have a trailing '\n'
594 594 f = f.rstrip()
595 595
596 596 # The objects in the file list have paths relative
597 597 # to $ROOT or to the base/ptch directory specified on
598 598 # the command line.
599 599 # If it's relative to $ROOT, we'll need to add back the
600 600 # root_`uname -p` goo we stripped off in fnFormat()
601 601 if os.path.exists(base + f) :
602 602 fn = f;
603 603 elif os.path.exists(base + "root_" + arch + "/" + f) :
604 604 fn = "root_" + arch + "/" + f
605 605 else :
606 606 base_present = False
607 607
608 608 if base_present :
609 609 if not os.path.exists(ptch + fn) :
610 610 ptch_present = False
611 611 else :
612 612 if os.path.exists(ptch + f) :
613 613 fn = f
614 614 elif os.path.exists(ptch + "root_" + arch + "/" + f) :
615 615 fn = "root_" + arch + "/" + f
616 616 else :
617 617 ptch_present = False
618 618
619 619 if os.path.islink(base + fn) : # ignore links
620 620 base_present = False
621 621 if os.path.islink(ptch + fn) :
622 622 ptch_present = False
623 623
624 624 if base_present and ptch_present :
625 625 compFiles.append(fn)
626 626 elif base_present :
627 627 deletedFiles.append(fn)
628 628 elif ptch_present :
629 629 newFiles.append(fn)
630 630 else :
631 631 if os.path.islink(base + fn) and \
632 632 os.path.islink(ptch + fn) :
633 633 continue
634 634 error(f + " in file list, but not in either tree. " + \
635 635 "Skipping...")
636 636
637 637 return compFiles, newFiles, deletedFiles
638 638
639 639
640 640 #
641 641 # Build a fully qualified path to an external tool/utility.
642 642 # Consider the default system locations. For onbld tools, if
643 643 # the -t option was specified, we'll try to use built tools in $SRC tools,
644 644 # and otherwise, we'll fall back on /opt/onbld/
645 645 #
646 646 def find_tool(tool) :
647 647
648 648 # First, check what was passed
649 649 if os.path.exists(tool) :
650 650 return tool
651 651
652 652 # Next try in wsdiff path
653 653 for pdir in wsdiff_path :
654 654 location = pdir + "/" + tool
655 655 if os.path.exists(location) :
656 656 return location + " "
657 657
658 658 location = pdir + "/" + arch + "/" + tool
659 659 if os.path.exists(location) :
660 660 return location + " "
661 661
662 662 error("Could not find path to: " + tool);
663 663 sys.exit(1);
664 664
665 665
666 666 #####
667 667 # ELF file comparison helper routines
668 668 #
669 669
670 670 #
671 671 # Return a dictionary of ELF section types keyed by section name
672 672 #
673 673 def get_elfheader(f) :
674 674
675 675 header = {}
676 676
677 677 hstring = commands.getoutput(elfdump_cmd + " -c " + f)
678 678
679 679 if len(hstring) == 0 :
680 680 error("Failed to dump ELF header for " + f)
681 681 raise
682 682 return
683 683
684 684 # elfdump(1) dumps the section headers with the section name
685 685 # following "sh_name:", and the section type following "sh_type:"
686 686 sections = hstring.split("Section Header")
687 687 for sect in sections :
688 688 datap = sect.find("sh_name:");
689 689 if datap == -1 :
690 690 continue
691 691 section = sect[datap:].split()[1]
692 692 datap = sect.find("sh_type:");
693 693 if datap == -1 :
694 694 error("Could not get type for sect: " + section + \
695 695 " in " + f)
696 696 sh_type = sect[datap:].split()[2]
697 697 header[section] = sh_type
698 698
699 699 return header
700 700
701 701 #
702 702 # Extract data in the specified ELF section from the given file
703 703 #
704 704 def extract_elf_section(f, section) :
705 705
706 706 data = commands.getoutput(dump_cmd + " -sn " + section + " " + f)
707 707
708 708 if len(data) == 0 :
709 709 error(dump_cmd + "yielded no data on section " + section + \
710 710 " of " + f)
711 711 raise
712 712 return
713 713
714 714 # dump(1) displays the file name to start...
715 715 # get past it to the data itself
716 716 dbegin = data.find(":") + 1
717 717 data = data[dbegin:];
718 718
719 719 return (data)
720 720
721 721 #
722 722 # Return a (hopefully meaningful) human readable set of diffs
723 723 # for the specified ELF section between f1 and f2
724 724 #
725 725 # Depending on the section, various means for dumping and diffing
726 726 # the data may be employed.
727 727 #
728 728 text_sections = [ '.text', '.init', '.fini' ]
729 729 def diff_elf_section(f1, f2, section, sh_type) :
730 730
731 731 t = threading.currentThread()
732 732 tmpFile1 = tmpDir1 + os.path.basename(f1) + t.getName()
733 733 tmpFile2 = tmpDir2 + os.path.basename(f2) + t.getName()
734 734
735 735 if (sh_type == "SHT_RELA") : # sh_type == SHT_RELA
736 736 cmd1 = elfdump_cmd + " -r " + f1 + " > " + tmpFile1
737 737 cmd2 = elfdump_cmd + " -r " + f2 + " > " + tmpFile2
738 738 elif (section == ".group") :
739 739 cmd1 = elfdump_cmd + " -g " + f1 + " > " + tmpFile1
740 740 cmd2 = elfdump_cmd + " -g " + f2 + " > " + tmpFile2
741 741 elif (section == ".hash") :
742 742 cmd1 = elfdump_cmd + " -h " + f1 + " > " + tmpFile1
743 743 cmd2 = elfdump_cmd + " -h " + f2 + " > " + tmpFile2
744 744 elif (section == ".dynamic") :
745 745 cmd1 = elfdump_cmd + " -d " + f1 + " > " + tmpFile1
746 746 cmd2 = elfdump_cmd + " -d " + f2 + " > " + tmpFile2
747 747 elif (section == ".got") :
748 748 cmd1 = elfdump_cmd + " -G " + f1 + " > " + tmpFile1
749 749 cmd2 = elfdump_cmd + " -G " + f2 + " > " + tmpFile2
750 750 elif (section == ".SUNW_cap") :
751 751 cmd1 = elfdump_cmd + " -H " + f1 + " > " + tmpFile1
752 752 cmd2 = elfdump_cmd + " -H " + f2 + " > " + tmpFile2
753 753 elif (section == ".interp") :
754 754 cmd1 = elfdump_cmd + " -i " + f1 + " > " + tmpFile1
755 755 cmd2 = elfdump_cmd + " -i " + f2 + " > " + tmpFile2
756 756 elif (section == ".symtab" or section == ".dynsym") :
757 757 cmd1 = elfdump_cmd + " -s -N " + section + " " + f1 + \
758 758 " > " + tmpFile1
759 759 cmd2 = elfdump_cmd + " -s -N " + section + " " + f2 + \
760 760 " > " + tmpFile2
761 761 elif (section in text_sections) :
762 762 # dis sometimes complains when it hits something it doesn't
763 763 # know how to disassemble. Just ignore it, as the output
764 764 # being generated here is human readable, and we've already
765 765 # correctly flagged the difference.
766 766 cmd1 = dis_cmd + " -t " + section + " " + f1 + \
767 767 " 2>/dev/null | grep -v disassembly > " + tmpFile1
768 768 cmd2 = dis_cmd + " -t " + section + " " + f2 + \
769 769 " 2>/dev/null | grep -v disassembly > " + tmpFile2
770 770 else :
771 771 cmd1 = elfdump_cmd + " -w " + tmpFile1 + " -N " + \
772 772 section + " " + f1
773 773 cmd2 = elfdump_cmd + " -w " + tmpFile2 + " -N " + \
774 774 section + " " + f2
775 775
776 776 os.system(cmd1)
777 777 os.system(cmd2)
778 778
779 779 data = diffFileData(tmpFile1, tmpFile2)
780 780
781 781 # remove temp files as we no longer need them
782 782 try:
783 783 os.unlink(tmpFile1)
784 784 except OSError, e:
785 785 error("diff_elf_section: unlink failed %s" % e)
786 786 try:
787 787 os.unlink(tmpFile2)
788 788 except OSError, e:
789 789 error("diff_elf_section: unlink failed %s" % e)
790 790
791 791 return (data)
792 792
793 793 #
794 794 # compare the relevant sections of two ELF binaries
795 795 # and report any differences
796 796 #
797 797 # Returns: 1 if any differenes found
798 798 # 0 if no differences found
799 799 # -1 on error
800 800 #
801 801
802 802 # Sections deliberately not considered when comparing two ELF
803 803 # binaries. Differences observed in these sections are not considered
804 804 # significant where patch deliverable identification is concerned.
805 805 sections_to_skip = [ ".SUNW_signature",
806 806 ".comment",
807 807 ".SUNW_ctf",
808 808 ".debug",
809 809 ".plt",
810 810 ".rela.bss",
811 811 ".rela.plt",
812 812 ".line",
813 813 ".note",
814 814 ".compcom",
815 815 ]
816 816
817 817 sections_preferred = [ ".rodata.str1.8",
818 818 ".rodata.str1.1",
819 819 ".rodata",
820 820 ".data1",
821 821 ".data",
822 822 ".text",
823 823 ]
824 824
825 825 def compareElfs(base, ptch, quiet) :
826 826
827 827 global logging
828 828
829 829 try:
830 830 base_header = get_elfheader(base)
831 831 except:
832 832 return
833 833 sections = base_header.keys()
834 834
835 835 try:
836 836 ptch_header = get_elfheader(ptch)
837 837 except:
838 838 return
839 839 e2_only_sections = ptch_header.keys()
840 840
841 841 e1_only_sections = []
842 842
843 843 fileName = fnFormat(base)
844 844
845 845 # Derive the list of ELF sections found only in
846 846 # either e1 or e2.
847 847 for sect in sections :
848 848 if not sect in e2_only_sections :
849 849 e1_only_sections.append(sect)
850 850 else :
851 851 e2_only_sections.remove(sect)
852 852
853 853 if len(e1_only_sections) > 0 :
854 854 if quiet :
855 855 return 1
856 856
857 857 data = ""
858 858 if logging :
859 859 slist = ""
860 860 for sect in e1_only_sections :
861 861 slist = slist + sect + "\t"
862 862 data = "ELF sections found in " + \
863 863 base + " but not in " + ptch + \
864 864 "\n\n" + slist
865 865
866 866 difference(fileName, "ELF", data)
867 867 return 1
868 868
869 869 if len(e2_only_sections) > 0 :
870 870 if quiet :
871 871 return 1
872 872
873 873 data = ""
874 874 if logging :
875 875 slist = ""
876 876 for sect in e2_only_sections :
877 877 slist = slist + sect + "\t"
878 878 data = "ELF sections found in " + \
879 879 ptch + " but not in " + base + \
880 880 "\n\n" + slist
881 881
882 882 difference(fileName, "ELF", data)
883 883 return 1
884 884
885 885 # Look for preferred sections, and put those at the
886 886 # top of the list of sections to compare
887 887 for psect in sections_preferred :
888 888 if psect in sections :
889 889 sections.remove(psect)
890 890 sections.insert(0, psect)
891 891
892 892 # Compare ELF sections
893 893 first_section = True
894 894 for sect in sections :
895 895
896 896 if sect in sections_to_skip :
897 897 continue
898 898
899 899 try:
900 900 s1 = extract_elf_section(base, sect);
901 901 except:
902 902 return
903 903
904 904 try:
905 905 s2 = extract_elf_section(ptch, sect);
906 906 except:
907 907 return
908 908
909 909 if len(s1) != len (s2) or s1 != s2:
910 910 if not quiet:
911 911 sh_type = base_header[sect]
912 912 data = diff_elf_section(base, ptch, \
913 913 sect, sh_type)
914 914
915 915 # If all ELF sections are being reported, then
916 916 # invoke difference() to flag the file name to
917 917 # stdout only once. Any other section differences
918 918 # should be logged to the results file directly
919 919 if not first_section :
920 920 log_difference(fileName, \
921 921 "ELF " + sect, data)
922 922 else :
923 923 difference(fileName, "ELF " + sect, \
924 924 data)
925 925
926 926 if not reportAllSects :
927 927 return 1
928 928 first_section = False
929 929
930 930 return 0
931 931
932 932 #####
933 933 # recursively remove 2 directories
934 934 #
935 935 # Used for removal of temporary directory strucures (ignores any errors).
936 936 #
937 937 def clearTmpDirs(dir1, dir2) :
938 938
939 939 if os.path.isdir(dir1) > 0 :
940 940 shutil.rmtree(dir1, True)
941 941
942 942 if os.path.isdir(dir2) > 0 :
943 943 shutil.rmtree(dir2, True)
944 944
945 945
946 946 #####
947 947 # Archive object comparison
948 948 #
949 949 # Returns 1 if difference detected
950 950 # 0 if no difference detected
951 951 # -1 on error
952 952 #
953 953 def compareArchives(base, ptch, fileType) :
954 954
955 955 fileName = fnFormat(base)
956 956 t = threading.currentThread()
957 957 ArchTmpDir1 = tmpDir1 + os.path.basename(base) + t.getName()
958 958 ArchTmpDir2 = tmpDir2 + os.path.basename(base) + t.getName()
959 959
960 960 #
961 961 # Be optimistic and first try a straight file compare
962 962 # as it will allow us to finish up quickly.
963 963 #
964 964 if compareBasic(base, ptch, True, fileType) == 0 :
965 965 return 0
966 966
967 967 try:
968 968 os.makedirs(ArchTmpDir1)
969 969 except OSError, e:
970 970 error("compareArchives: makedir failed %s" % e)
971 971 return -1
972 972 try:
973 973 os.makedirs(ArchTmpDir2)
974 974 except OSError, e:
975 975 error("compareArchives: makedir failed %s" % e)
976 976 return -1
977 977
978 978 # copy over the objects to the temp areas, and
979 979 # unpack them
980 980 baseCmd = "cp -fp " + base + " " + ArchTmpDir1
981 981 status, output = commands.getstatusoutput(baseCmd)
982 982 if status != 0 :
983 983 error(baseCmd + " failed: " + output)
984 984 clearTmpDirs(ArchTmpDir1, ArchTmpDir2)
985 985 return -1
986 986
987 987 ptchCmd = "cp -fp " + ptch + " " + ArchTmpDir2
988 988 status, output = commands.getstatusoutput(ptchCmd)
989 989 if status != 0 :
990 990 error(ptchCmd + " failed: " + output)
991 991 clearTmpDirs(ArchTmpDir1, ArchTmpDir2)
992 992 return -1
993 993
994 994 bname = string.split(fileName, '/')[-1]
995 995 if fileType == "Java Archive" :
996 996 baseCmd = "cd " + ArchTmpDir1 + "; " + "jar xf " + bname + \
997 997 "; rm -f " + bname + " META-INF/MANIFEST.MF"
998 998 ptchCmd = "cd " + ArchTmpDir2 + "; " + "jar xf " + bname + \
999 999 "; rm -f " + bname + " META-INF/MANIFEST.MF"
1000 1000 elif fileType == "ELF Object Archive" :
1001 1001 baseCmd = "cd " + ArchTmpDir1 + "; " + "/usr/ccs/bin/ar x " + \
1002 1002 bname + "; rm -f " + bname
1003 1003 ptchCmd = "cd " + ArchTmpDir2 + "; " + "/usr/ccs/bin/ar x " + \
1004 1004 bname + "; rm -f " + bname
1005 1005 else :
1006 1006 error("unexpected file type: " + fileType)
1007 1007 clearTmpDirs(ArchTmpDir1, ArchTmpDir2)
1008 1008 return -1
1009 1009
1010 1010 os.system(baseCmd)
1011 1011 os.system(ptchCmd)
1012 1012
1013 1013 baseFlist = list(findFiles(ArchTmpDir1))
1014 1014 ptchFlist = list(findFiles(ArchTmpDir2))
1015 1015
1016 1016 # Trim leading path off base/ptch file lists
1017 1017 flist = []
1018 1018 for fn in baseFlist :
1019 1019 flist.append(str_prefix_trunc(fn, ArchTmpDir1))
1020 1020 baseFlist = flist
1021 1021
1022 1022 flist = []
1023 1023 for fn in ptchFlist :
1024 1024 flist.append(str_prefix_trunc(fn, ArchTmpDir2))
1025 1025 ptchFlist = flist
1026 1026
1027 1027 for fn in ptchFlist :
1028 1028 if not fn in baseFlist :
1029 1029 difference(fileName, fileType, \
1030 1030 fn + " added to " + fileName)
1031 1031 clearTmpDirs(ArchTmpDir1, ArchTmpDir2)
1032 1032 return 1
1033 1033
1034 1034 for fn in baseFlist :
1035 1035 if not fn in ptchFlist :
1036 1036 difference(fileName, fileType, \
1037 1037 fn + " removed from " + fileName)
1038 1038 clearTmpDirs(ArchTmpDir1, ArchTmpDir2)
1039 1039 return 1
1040 1040
1041 1041 differs = compareOneFile((ArchTmpDir1 + fn), \
1042 1042 (ArchTmpDir2 + fn), True)
1043 1043 if differs :
1044 1044 difference(fileName, fileType, \
1045 1045 fn + " in " + fileName + " differs")
1046 1046 clearTmpDirs(ArchTmpDir1, ArchTmpDir2)
1047 1047 return 1
1048 1048
1049 1049 clearTmpDirs(ArchTmpDir1, ArchTmpDir2)
1050 1050 return 0
1051 1051
1052 1052 #####
1053 1053 # (Basic) file comparison
1054 1054 #
1055 1055 # There's some special case code here for Javadoc HTML files
1056 1056 #
1057 1057 # Returns 1 if difference detected
1058 1058 # 0 if no difference detected
1059 1059 # -1 on error
1060 1060 #
1061 1061 def compareBasic(base, ptch, quiet, fileType) :
1062 1062
1063 1063 fileName = fnFormat(base);
1064 1064
1065 1065 if quiet and os.stat(base)[ST_SIZE] != os.stat(ptch)[ST_SIZE] :
1066 1066 return 1
1067 1067
1068 1068 try:
1069 1069 baseFile = open(base)
1070 1070 except:
1071 1071 error("could not open " + base)
1072 1072 return -1
1073 1073 try:
1074 1074 ptchFile = open(ptch)
1075 1075 except:
1076 1076 error("could not open " + ptch)
1077 1077 return -1
1078 1078
1079 1079 baseData = baseFile.read()
1080 1080 ptchData = ptchFile.read()
1081 1081
1082 1082 baseFile.close()
1083 1083 ptchFile.close()
1084 1084
1085 1085 needToSnip = False
1086 1086 if fileType == "HTML" :
1087 1087 needToSnip = True
1088 1088 toSnipBeginStr = "<!-- Generated by javadoc"
1089 1089 toSnipEndStr = "-->\n"
1090 1090
1091 1091 if needToSnip :
1092 1092 toSnipBegin = string.find(baseData, toSnipBeginStr)
1093 1093 if toSnipBegin != -1 :
1094 1094 toSnipEnd = string.find(baseData[toSnipBegin:], \
1095 1095 toSnipEndStr) + \
1096 1096 len(toSnipEndStr)
1097 1097 baseData = baseData[:toSnipBegin] + \
1098 1098 baseData[toSnipBegin + toSnipEnd:]
1099 1099 ptchData = ptchData[:toSnipBegin] + \
1100 1100 ptchData[toSnipBegin + toSnipEnd:]
1101 1101
1102 1102 if quiet :
1103 1103 if baseData != ptchData :
1104 1104 return 1
1105 1105 else :
1106 1106 if len(baseData) != len(ptchData) or baseData != ptchData :
1107 1107 diffs = diffData(base, ptch, baseData, ptchData)
1108 1108 difference(fileName, fileType, diffs)
1109 1109 return 1
1110 1110 return 0
1111 1111
1112 1112
1113 1113 #####
1114 1114 # Compare two objects by producing a data dump from
1115 1115 # each object, and then comparing the dump data
1116 1116 #
1117 1117 # Returns: 1 if a difference is detected
1118 1118 # 0 if no difference detected
1119 1119 # -1 upon error
1120 1120 #
1121 1121 def compareByDumping(base, ptch, quiet, fileType) :
1122 1122
1123 1123 fileName = fnFormat(base);
1124 1124 t = threading.currentThread()
1125 1125 tmpFile1 = tmpDir1 + os.path.basename(base) + t.getName()
1126 1126 tmpFile2 = tmpDir2 + os.path.basename(ptch) + t.getName()
1127 1127
1128 1128 if fileType == "Lint Library" :
1129 1129 baseCmd = lintdump_cmd + " -ir " + base + \
1130 1130 " | egrep -v '(LINTOBJ|LINTMOD):'" + \
1131 1131 " | grep -v PASS[1-3]:" + \
1132 1132 " > " + tmpFile1
1133 1133 ptchCmd = lintdump_cmd + " -ir " + ptch + \
1134 1134 " | egrep -v '(LINTOBJ|LINTMOD):'" + \
1135 1135 " | grep -v PASS[1-3]:" + \
1136 1136 " > " + tmpFile2
1137 1137 elif fileType == "Sqlite Database" :
1138 1138 baseCmd = "echo .dump | " + sqlite_cmd + base + " > " + \
1139 1139 tmpFile1
1140 1140 ptchCmd = "echo .dump | " + sqlite_cmd + ptch + " > " + \
1141 1141 tmpFile2
1142 1142
1143 1143 os.system(baseCmd)
1144 1144 os.system(ptchCmd)
1145 1145
1146 1146 try:
1147 1147 baseFile = open(tmpFile1)
1148 1148 except:
1149 1149 error("could not open: " + tmpFile1)
1150 1150 return
1151 1151 try:
1152 1152 ptchFile = open(tmpFile2)
1153 1153 except:
1154 1154 error("could not open: " + tmpFile2)
1155 1155 return
1156 1156
1157 1157 baseData = baseFile.read()
1158 1158 ptchData = ptchFile.read()
1159 1159
1160 1160 baseFile.close()
1161 1161 ptchFile.close()
1162 1162
1163 1163 if len(baseData) != len(ptchData) or baseData != ptchData :
1164 1164 if not quiet :
1165 1165 data = diffFileData(tmpFile1, tmpFile2);
1166 1166 try:
1167 1167 os.unlink(tmpFile1)
1168 1168 except OSError, e:
1169 1169 error("compareByDumping: unlink failed %s" % e)
1170 1170 try:
1171 1171 os.unlink(tmpFile2)
1172 1172 except OSError, e:
1173 1173 error("compareByDumping: unlink failed %s" % e)
1174 1174 difference(fileName, fileType, data)
1175 1175 return 1
1176 1176
1177 1177 # Remove the temporary files now.
1178 1178 try:
1179 1179 os.unlink(tmpFile1)
1180 1180 except OSError, e:
1181 1181 error("compareByDumping: unlink failed %s" % e)
1182 1182 try:
1183 1183 os.unlink(tmpFile2)
1184 1184 except OSError, e:
1185 1185 error("compareByDumping: unlink failed %s" % e)
1186 1186
1187 1187 return 0
1188 1188
1189 1189 #####
1190 1190 #
1191 1191 # SIGINT signal handler. Changes thread control variable to tell the threads
1192 1192 # to finish their current job and exit.
1193 1193 #
1194 1194 def discontinue_processing(signl, frme):
1195 1195 global keep_processing
1196 1196
1197 1197 print >> sys.stderr, "Caught Ctrl-C, stopping the threads"
1198 1198 keep_processing = False
1199 1199
1200 1200 return 0
1201 1201
1202 1202 #####
1203 1203 #
1204 1204 # worker thread for changedFiles processing
1205 1205 #
1206 1206 class workerThread(threading.Thread) :
1207 1207 def run(self):
1208 1208 global wset_lock
1209 1209 global changedFiles
1210 1210 global baseRoot
1211 1211 global ptchRoot
1212 1212 global keep_processing
1213 1213
1214 1214 while (keep_processing) :
1215 1215 # grab the lock to changedFiles and remove one member
1216 1216 # and process it
1217 1217 wset_lock.acquire()
1218 1218 try :
1219 1219 fn = changedFiles.pop()
1220 1220 except IndexError :
1221 1221 # there is nothing more to do
1222 1222 wset_lock.release()
1223 1223 return
1224 1224 wset_lock.release()
1225 1225
1226 1226 base = baseRoot + fn
1227 1227 ptch = ptchRoot + fn
1228 1228
1229 1229 compareOneFile(base, ptch, False)
1230 1230
1231 1231
1232 1232 #####
1233 1233 # Compare two objects. Detect type changes.
1234 1234 # Vector off to the appropriate type specific
1235 1235 # compare routine based on the type.
1236 1236 #
1237 1237 def compareOneFile(base, ptch, quiet) :
1238 1238
1239 1239 # Verify the file types.
1240 1240 # If they are different, indicate this and move on
1241 1241 btype = getTheFileType(base)
1242 1242 ptype = getTheFileType(ptch)
1243 1243
1244 1244 if btype == 'Error' or ptype == 'Error' :
1245 1245 return -1
1246 1246
1247 1247 fileName = fnFormat(base)
1248 1248
1249 1249 if (btype != ptype) :
1250 1250 if not quiet :
1251 1251 difference(fileName, "file type", btype + " to " + ptype)
1252 1252 return 1
1253 1253 else :
1254 1254 fileType = btype
1255 1255
1256 1256 if (fileType == 'ELF') :
1257 1257 return compareElfs(base, ptch, quiet)
1258 1258
1259 1259 elif (fileType == 'Java Archive' or fileType == 'ELF Object Archive') :
1260 1260 return compareArchives(base, ptch, fileType)
1261 1261
1262 1262 elif (fileType == 'HTML') :
1263 1263 return compareBasic(base, ptch, quiet, fileType)
1264 1264
1265 1265 elif ( fileType == 'Lint Library' ) :
1266 1266 return compareByDumping(base, ptch, quiet, fileType)
1267 1267
1268 1268 elif ( fileType == 'Sqlite Database' ) :
1269 1269 return compareByDumping(base, ptch, quiet, fileType)
1270 1270
1271 1271 else :
1272 1272 # it has to be some variety of text file
1273 1273 return compareBasic(base, ptch, quiet, fileType)
1274 1274
1275 1275 # Cleanup and self-terminate
1276 1276 def cleanup(ret) :
1277 1277
1278 1278 debug("Performing cleanup (" + str(ret) + ")")
1279 1279 if os.path.isdir(tmpDir1) > 0 :
1280 1280 shutil.rmtree(tmpDir1)
1281 1281
1282 1282 if os.path.isdir(tmpDir2) > 0 :
1283 1283 shutil.rmtree(tmpDir2)
1284 1284
1285 1285 if logging :
1286 1286 log.close()
1287 1287
1288 1288 sys.exit(ret)
1289 1289
1290 1290 def main() :
1291 1291
1292 1292 # Log file handle
1293 1293 global log
1294 1294
1295 1295 # Globals relating to command line options
1296 1296 global logging, vdiffs, reportAllSects
1297 1297
1298 1298 # Named temporary files / directories
1299 1299 global tmpDir1, tmpDir2
1300 1300
1301 1301 # Command paths
1302 1302 global lintdump_cmd, elfdump_cmd, dump_cmd, dis_cmd, od_cmd, diff_cmd, sqlite_cmd
1303 1303
1304 1304 # Default search path
1305 1305 global wsdiff_path
1306 1306
1307 1307 # Essentially "uname -p"
1308 1308 global arch
1309 1309
1310 1310 # changed files for worker thread processing
1311 1311 global changedFiles
1312 1312 global baseRoot
1313 1313 global ptchRoot
1314 1314
1315 1315 # Sort the list of files from a temporary file
1316 1316 global sorted
1317 1317 global differentFiles
1318 1318
1319 1319 # Debugging indicator
1320 1320 global debugon
1321 1321
1322 1322 # Some globals need to be initialized
1323 1323 debugon = logging = vdiffs = reportAllSects = sorted = False
1324 1324
1325 1325
1326 1326 # Process command line arguments
1327 1327 # Return values are returned from args() in alpha order
1328 1328 # (Yes, python functions can return multiple values (ewww))
1329 1329 # Note that args() also set the globals:
1330 1330 # logging to True if verbose logging (to a file) was enabled
1331 1331 # vdiffs to True if logged differences aren't to be truncated
1332 1332 # reportAllSects to True if all ELF section differences are to be reported
1333 1333 #
1334 1334 baseRoot, fileNamesFile, localTools, ptchRoot, results = args()
1335 1335
1336 1336 #
|
↓ open down ↓ |
1336 lines elided |
↑ open up ↑ |
1337 1337 # Set up the results/log file
1338 1338 #
1339 1339 if logging :
1340 1340 try:
1341 1341 log = open(results, "w")
1342 1342 except:
1343 1343 logging = False
1344 1344 error("failed to open log file: " + log)
1345 1345 sys.exit(1)
1346 1346
1347 - dateTimeStr= "# %d/%d/%d at %d:%d:%d" % time.localtime()[:6]
1347 + dateTimeStr= "# %04d-%02d-%02d at %02d:%02d:%02d" % time.localtime()[:6]
1348 1348 v_info("# This file was produced by wsdiff")
1349 1349 v_info(dateTimeStr)
1350 1350
1351 1351 # Changed files (used only for the sorted case)
1352 1352 if sorted :
1353 1353 differentFiles = []
1354 1354
1355 1355 #
1356 1356 # Build paths to the tools required tools
1357 1357 #
1358 1358 # Try to look for tools in $SRC/tools if the "-t" option
1359 1359 # was specified
1360 1360 #
1361 1361 arch = commands.getoutput("uname -p")
1362 1362 if localTools :
1363 1363 try:
1364 1364 src = os.environ['SRC']
1365 1365 except:
1366 1366 error("-t specified, but $SRC not set. Cannot find $SRC/tools")
1367 1367 src = ""
1368 1368 if len(src) > 0 :
1369 1369 wsdiff_path.insert(0, src + "/tools/proto/opt/onbld/bin")
1370 1370
1371 1371 lintdump_cmd = find_tool("lintdump")
1372 1372 elfdump_cmd = find_tool("elfdump")
1373 1373 dump_cmd = find_tool("dump")
1374 1374 od_cmd = find_tool("od")
1375 1375 dis_cmd = find_tool("dis")
1376 1376 diff_cmd = find_tool("diff")
1377 1377 sqlite_cmd = find_tool("sqlite")
1378 1378
1379 1379 #
1380 1380 # Set resource limit for number of open files as high as possible.
1381 1381 # This might get handy with big number of threads.
1382 1382 #
1383 1383 (nofile_soft, nofile_hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
1384 1384 try:
1385 1385 resource.setrlimit(resource.RLIMIT_NOFILE,
1386 1386 (nofile_hard, nofile_hard))
1387 1387 except:
1388 1388 error("cannot set resource limits for number of open files")
1389 1389 sys.exit(1)
1390 1390
1391 1391 #
1392 1392 # validate the base and patch paths
1393 1393 #
1394 1394 if baseRoot[-1] != '/' :
1395 1395 baseRoot += '/'
1396 1396
1397 1397 if ptchRoot[-1] != '/' :
1398 1398 ptchRoot += '/'
1399 1399
1400 1400 if not os.path.exists(baseRoot) :
1401 1401 error("old proto area: " + baseRoot + " does not exist")
1402 1402 sys.exit(1)
1403 1403
1404 1404 if not os.path.exists(ptchRoot) :
1405 1405 error("new proto area: " + ptchRoot + \
1406 1406 " does not exist")
1407 1407 sys.exit(1)
1408 1408
1409 1409 #
1410 1410 # log some information identifying the run
1411 1411 #
1412 1412 v_info("Old proto area: " + baseRoot)
1413 1413 v_info("New proto area: " + ptchRoot)
1414 1414 v_info("Results file: " + results + "\n")
1415 1415
1416 1416 #
1417 1417 # Set up the temporary directories / files
1418 1418 # Could use python's tmpdir routines, but these should
1419 1419 # be easier to identify / keep around for debugging
1420 1420 pid = os.getpid()
1421 1421 tmpDir1 = "/tmp/wsdiff_tmp1_" + str(pid) + "/"
1422 1422 tmpDir2 = "/tmp/wsdiff_tmp2_" + str(pid) + "/"
1423 1423 try:
1424 1424 os.makedirs(tmpDir1)
1425 1425 except OSError, e:
1426 1426 error("main: makedir failed %s" % e)
1427 1427 try:
1428 1428 os.makedirs(tmpDir2)
1429 1429 except OSError, e:
1430 1430 error("main: makedir failed %s" % e)
1431 1431
1432 1432 # Derive a catalog of new, deleted, and to-be-compared objects
1433 1433 # either from the specified base and patch proto areas, or from
1434 1434 # from an input file list
1435 1435 newOrDeleted = False
1436 1436
1437 1437 if fileNamesFile != "" :
1438 1438 changedFiles, newFiles, deletedFiles = \
1439 1439 flistCatalog(baseRoot, ptchRoot, fileNamesFile)
1440 1440 else :
1441 1441 changedFiles, newFiles, deletedFiles = \
1442 1442 protoCatalog(baseRoot, ptchRoot)
1443 1443
1444 1444 if len(newFiles) > 0 :
1445 1445 newOrDeleted = True
1446 1446 info("\nNew objects found: ")
1447 1447
1448 1448 if sorted :
1449 1449 newFiles.sort()
1450 1450 for fn in newFiles :
1451 1451 info(fnFormat(fn))
1452 1452
1453 1453 if len(deletedFiles) > 0 :
1454 1454 newOrDeleted = True
1455 1455 info("\nObjects removed: ")
1456 1456
1457 1457 if sorted :
1458 1458 deletedFiles.sort()
1459 1459 for fn in deletedFiles :
1460 1460 info(fnFormat(fn))
1461 1461
1462 1462 if newOrDeleted :
1463 1463 info("\nChanged objects: ")
1464 1464 if sorted :
1465 1465 debug("The list will appear after the processing is done")
1466 1466
1467 1467 # Here's where all the heavy lifting happens
1468 1468 # Perform a comparison on each object appearing in
1469 1469 # both proto areas. compareOneFile will examine the
1470 1470 # file types of each object, and will vector off to
1471 1471 # the appropriate comparison routine, where the compare
1472 1472 # will happen, and any differences will be reported / logged
1473 1473
1474 1474 # determine maximum number of worker threads by using
1475 1475 # DMAKE_MAX_JOBS environment variable set by nightly(1)
1476 1476 # or get number of CPUs in the system
1477 1477 try:
1478 1478 max_threads = int(os.environ['DMAKE_MAX_JOBS'])
1479 1479 except:
1480 1480 max_threads = os.sysconf("SC_NPROCESSORS_ONLN")
1481 1481 # If we cannot get number of online CPUs in the system
1482 1482 # run unparallelized otherwise bump the number up 20%
1483 1483 # to achieve best results.
1484 1484 if max_threads == -1 :
1485 1485 max_threads = 1
1486 1486 else :
1487 1487 max_threads += max_threads/5
1488 1488
1489 1489 # Set signal handler to attempt graceful exit
1490 1490 debug("Setting signal handler")
1491 1491 signal.signal( signal.SIGINT, discontinue_processing )
1492 1492
1493 1493 # Create and unleash the threads
1494 1494 # Only at most max_threads must be running at any moment
1495 1495 mythreads = []
1496 1496 debug("Spawning " + str(max_threads) + " threads");
1497 1497 for i in range(max_threads) :
1498 1498 thread = workerThread()
1499 1499 mythreads.append(thread)
1500 1500 mythreads[i].start()
1501 1501
1502 1502 # Wait for the threads to finish and do cleanup if interrupted
1503 1503 debug("Waiting for the threads to finish")
1504 1504 while True:
1505 1505 if not True in [thread.isAlive() for thread in mythreads]:
1506 1506 break
1507 1507 else:
1508 1508 # Some threads are still going
1509 1509 time.sleep(1)
1510 1510
1511 1511 # Interrupted by SIGINT
1512 1512 if keep_processing == False :
1513 1513 cleanup(1)
1514 1514
1515 1515 # If the list of differences was sorted it is stored in an array
1516 1516 if sorted :
1517 1517 differentFiles.sort()
1518 1518 for f in differentFiles :
1519 1519 info(fnFormat(f))
1520 1520
1521 1521 # We're done, cleanup.
1522 1522 cleanup(0)
1523 1523
1524 1524 if __name__ == '__main__' :
1525 1525 try:
1526 1526 main()
1527 1527 except KeyboardInterrupt :
1528 1528 cleanup(1);
1529 1529
1530 1530
|
↓ open down ↓ |
173 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX