dep_tree2sql.sh 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583
  1. #!/bin/bash
  2. ############################################################################
  3. #
  4. # MODULE: dep_tree2sql.sh
  5. # AUTHOR(S): Glynn Clements
  6. # PURPOSE: run "nm" on every object file, library and executable to
  7. # find dependencies, stored in PostgreSQL database
  8. # COPYRIGHT: (C) 2006 by the GRASS Development Team
  9. #
  10. # This program is free software under the GNU General Public
  11. # License (>=v2). Read the file COPYING that comes with GRASS
  12. # for details.
  13. #
  14. #############################################################################
  15. #
  16. # From: Glynn Clements <glynn at gclements.plus.com>
  17. # Date: Mon, 12 Jun 2006 13:00:25 +0100
  18. #
  19. # Usage: after having compiled GRASS, run "tools/dep_tree2sql.sh `pwd`" from the
  20. # top of the GRASS source tree.
  21. #
  22. # Essentially, the script runs "nm" on every object file, library and
  23. # executable it finds (and "ldd" on all of the executables), processes
  24. # the output with egrep/sed/awk, then imports the results into a
  25. # PostgreSQL database.
  26. #
  27. # "nm" lists the symbol table of an object file, library, or executable,
  28. # indicating whether each symbol is imported into, defined in and/or
  29. # exported from the file.
  30. #
  31. # Most of the database tables record which symbols are imported into or
  32. # exported from which object files, libraries or executables. E.g. the
  33. # "obj_imp" table lists which symbols are imported into which object
  34. # files.
  35. #
  36. # You can then use simple queries such as:
  37. #
  38. # grass=> SELECT object FROM obj_imp WHERE symbol = 'I_get_target' ;
  39. # object
  40. # --------------------------------------------------------------------
  41. # imagery/i.ortho.photo/photo.2image/OBJ.i686-pc-linux-gnu/target.o
  42. # imagery/i.ortho.photo/photo.2target/OBJ.i686-pc-linux-gnu/target.o
  43. # imagery/i.ortho.photo/photo.elev/OBJ.i686-pc-linux-gnu/main.o
  44. # imagery/i.ortho.photo/photo.rectify/OBJ.i686-pc-linux-gnu/target.o
  45. # imagery/i.ortho.photo/photo.target/OBJ.i686-pc-linux-gnu/main.o
  46. # imagery/i.rectify/OBJ.i686-pc-linux-gnu/target.o
  47. #
  48. # to discover which files import a given symbol, or more complex queries
  49. # such as:
  50. #
  51. # grass=> SELECT DISTINCT b.object FROM lib_exp a, obj_imp b
  52. # grass-> WHERE a.library = 'libgrass_form.6.1.cvs.so' AND a.symbol = b.symbol ;
  53. # object
  54. # -----------------------------------------------------------
  55. # vector/v.digit/OBJ.i686-pc-linux-gnu/attr.o
  56. # vector/v.digit/OBJ.i686-pc-linux-gnu/line.o
  57. # vector/v.what/OBJ.i686-pc-linux-gnu/what.o
  58. # visualization/nviz/src/OBJ.i686-pc-linux-gnu/query_vect.o
  59. # (5 rows)
  60. #
  61. # to discover which files import any symbol defined in a specific
  62. # library. And so on.
  63. #
  64. # For simple "which files use this function" queries, a database lookup
  65. # is quicker and more reliable than grep-ing the source tree.
  66. #
  67. # Assuming that the dep_tree2sql.sh script runs successfully (some of it is
  68. # Linux-specific, other bits are PostgreSQL-specific, but the changes
  69. # required for a different OS or RDBMS should be quite minor), the
  70. # easiest way to figure out what is in a given table (apart from looking
  71. # at the name) is to just sample it, e.g.:
  72. #
  73. # grass=> SELECT * FROM stlib_exp LIMIT 5 ;
  74. # library | object | symbol
  75. # -------------------+------------+---------------
  76. # libgrass_manage.a | add_elem.o | add_element
  77. # libgrass_manage.a | ask.o | ask_in_mapset
  78. # libgrass_manage.a | ask.o | ask_new
  79. # libgrass_manage.a | ask.o | ask_old
  80. # libgrass_manage.a | copyfile.o | copyfile
  81. # (5 rows)
  82. #
  83. tmpdir=/tmp/sql-grass
  84. dbname=grass
  85. if [ -n "$1" ] ; then
  86. builddir="$1"
  87. else
  88. echo "Usage: del_tree2sql.sh <source directory>" >&2
  89. exit 1
  90. fi
  91. rm -rf "$tmpdir"
  92. mkdir -m 711 "$tmpdir" || exit 1
  93. cd $builddir
  94. ( cd dist.*
  95. #LD_LIBRARY_PATH=`pwd`/lib
  96. #export LD_LIBRARY_PATH
  97. find . -type f -perm +111 \! -name '*.so.*' \
  98. | while read file ; do ldd $file | sed 's!^!'$file'!' ; done 2>/dev/null \
  99. | sed -e 's/^\.\///' -e 's/ (0x.*)$//' -e 's/ => \(.*\)$/ \1/' -e 's/ => .*$//' \
  100. | fgrep -v 'not a dynamic executable' \
  101. | awk -vOFS='\t' '{print $1,$2,$3 ? $3 : $2}' \
  102. > "$tmpdir/ldd.lst"
  103. find . -type f -perm +111 \! -name '*.so' \
  104. | xargs nm -AD 2>/dev/null \
  105. | egrep ': {8}{1,2} U ' \
  106. | sed -e 's/:/ /g' -e 's/\.\///' \
  107. | awk -vOFS='\t' '{print $1,$3}' \
  108. > "$tmpdir/prog_imp.lst"
  109. find . -type f -perm +111 \! -name '*.so' \
  110. | xargs nm -AD 2>/dev/null \
  111. | egrep ':[0-9a-f]{8}{1,2} [BCDGRSTW] ' \
  112. | sed -e 's/:/ /g' -e 's/\.\///' \
  113. | awk -vOFS='\t' '{print $1,$4}' \
  114. > "$tmpdir/prog_exp.lst"
  115. )
  116. find * -type f -name 'lib?*.a' \
  117. | xargs nm -A \
  118. | egrep ':[0-9a-f]{8}{1,2} [BCDGRSTW] ' \
  119. | sed 's/:/ /g' \
  120. | awk -vOFS='\t' '{print gensub("^[^ ]*/","",1,$1),$2,$5}' \
  121. > "$tmpdir/stlib_exp.lst"
  122. find * -type f -name 'lib?*.so' \
  123. | xargs nm -AD \
  124. | egrep ':[0-9a-f]{8}{1,2} [BCDGRSTW] ' \
  125. | sed 's/:/ /g' \
  126. | awk -vOFS='\t' '{print gensub("^[^ ]*/","",1,$1),$4}' \
  127. > "$tmpdir/shlib_exp.lst"
  128. find * -type f -name '*.o' \
  129. | xargs nm -A \
  130. | egrep ':[0-9a-f]{8}{1,2} [BCDGRSTW] ' \
  131. | sed 's/:/ /g' \
  132. | awk -vOFS='\t' '{print $1,$4}' \
  133. > "$tmpdir/obj_exp.lst"
  134. find * -type f -name 'lib?*.a' \
  135. | xargs nm -A \
  136. | egrep ': {8}{1,2} U ' \
  137. | sed 's/:/ /g' \
  138. | awk -vOFS='\t' '{print gensub("^[^ ]*/","",1,$1),$2,$4}' \
  139. > "$tmpdir/stlib_imp.lst"
  140. find * -type f -name 'lib?*.so' \
  141. | xargs nm -AD \
  142. | egrep ': {8}{1,2} U ' \
  143. | sed 's/:/ /g' \
  144. | awk -vOFS='\t' '{print gensub("^[^ ]*/","",1,$1),$3}' \
  145. > "$tmpdir/shlib_imp.lst"
  146. find * -type f -name '*.o' \
  147. | xargs nm -A \
  148. | egrep ': {8}{1,2} U ' \
  149. | sed 's/:/ /g' \
  150. | awk -vOFS='\t' '{print $1,$3}' \
  151. > "$tmpdir/obj_imp.lst"
  152. libs=`awk '{print $3}' "$tmpdir/ldd.lst" | uniq | sort | uniq`
  153. nm -AD $libs \
  154. | egrep ':[0-9a-f]{8}{1,2} [TWDRC] ' \
  155. | sed 's/:/ /g' \
  156. | awk -vOFS='\t' '{print gensub("^[^ ]*/","",1,$1),$4}' \
  157. > "$tmpdir/libs.lst"
  158. cat > "$tmpdir/ansi.lst" <<EOF
  159. abort
  160. asctime
  161. atexit
  162. atof
  163. atoi
  164. atol
  165. bsearch
  166. calloc
  167. clearerr
  168. ctime
  169. exit
  170. fclose
  171. feof
  172. ferror
  173. fflush
  174. fgetc
  175. fgetpos
  176. fgets
  177. fopen
  178. fprintf
  179. fputc
  180. fputs
  181. fread
  182. free
  183. freopen
  184. fscanf
  185. fseek
  186. fsetpos
  187. ftell
  188. fwrite
  189. getc
  190. getchar
  191. getenv
  192. gets
  193. isalnum
  194. isalpha
  195. isdigit
  196. islower
  197. isspace
  198. ldiv
  199. localtime
  200. longjmp
  201. malloc
  202. memcmp
  203. memcpy
  204. memmove
  205. memset
  206. perror
  207. printf
  208. putc
  209. putchar
  210. puts
  211. qsort
  212. rand
  213. realloc
  214. remove
  215. rename
  216. rewind
  217. scanf
  218. setbuf
  219. setjmp
  220. setvbuf
  221. signal
  222. sprintf
  223. srand
  224. sscanf
  225. stderr
  226. stdin
  227. stdout
  228. strcat
  229. strchr
  230. strcmp
  231. strcpy
  232. strcspn
  233. strerror
  234. strftime
  235. strlen
  236. strncat
  237. strncmp
  238. strncpy
  239. strpbrk
  240. strrchr
  241. strspn
  242. strstr
  243. strtod
  244. strtok
  245. strtol
  246. time
  247. tmpfile
  248. tmpnam
  249. tolower
  250. toupper
  251. ungetc
  252. vfprintf
  253. vsprintf
  254. fgetpos64
  255. fopen64
  256. freopen64
  257. fsetpos64
  258. tmpfile64
  259. EOF
  260. dropdb "$dbname"
  261. createdb "$dbname"
  262. psql -n -q -d "$dbname" << EOF
  263. -- ----------------------------------------------------------------------
  264. CREATE TABLE stlib_exp (
  265. library VARCHAR(80) NOT NULL,
  266. object VARCHAR(40) NOT NULL,
  267. symbol VARCHAR(256) NOT NULL
  268. ) ;
  269. \copy stlib_exp FROM '$tmpdir/stlib_exp.lst'
  270. CREATE TABLE shlib_exp (
  271. library VARCHAR(80) NOT NULL,
  272. symbol VARCHAR(256) NOT NULL
  273. ) ;
  274. \copy shlib_exp FROM '$tmpdir/shlib_exp.lst'
  275. CREATE TABLE obj_exp (
  276. object VARCHAR(100) NOT NULL,
  277. symbol VARCHAR(256) NOT NULL
  278. ) ;
  279. \copy obj_exp FROM '$tmpdir/obj_exp.lst'
  280. CREATE TABLE stlib_imp (
  281. library VARCHAR(80) NOT NULL,
  282. object VARCHAR(40) NOT NULL,
  283. symbol VARCHAR(256) NOT NULL
  284. ) ;
  285. \copy stlib_imp FROM '$tmpdir/stlib_imp.lst'
  286. CREATE TABLE shlib_imp (
  287. library VARCHAR(80) NOT NULL,
  288. symbol VARCHAR(256) NOT NULL
  289. ) ;
  290. \copy shlib_imp FROM '$tmpdir/shlib_imp.lst'
  291. CREATE TABLE obj_imp (
  292. object VARCHAR(100) NOT NULL,
  293. symbol VARCHAR(256) NOT NULL
  294. ) ;
  295. \copy obj_imp FROM '$tmpdir/obj_imp.lst'
  296. CREATE TABLE prog_imp (
  297. program VARCHAR(80) NOT NULL,
  298. symbol VARCHAR(256) NOT NULL
  299. ) ;
  300. \copy prog_imp FROM '$tmpdir/prog_imp.lst'
  301. CREATE TABLE prog_exp (
  302. program VARCHAR(80) NOT NULL,
  303. symbol VARCHAR(256) NOT NULL
  304. ) ;
  305. \copy prog_exp FROM '$tmpdir/prog_exp.lst'
  306. CREATE TABLE libs (
  307. library VARCHAR(80) NOT NULL,
  308. symbol VARCHAR(1000) NOT NULL
  309. ) ;
  310. \copy libs FROM '$tmpdir/libs.lst'
  311. CREATE TABLE ldd (
  312. program VARCHAR(80) NOT NULL,
  313. library VARCHAR(256) NOT NULL,
  314. path VARCHAR(256)
  315. ) ;
  316. \copy ldd FROM '$tmpdir/ldd.lst'
  317. CREATE TABLE ansi (
  318. symbol VARCHAR(256) NOT NULL
  319. ) ;
  320. \copy ansi FROM '$tmpdir/ansi.lst'
  321. -- ----------------------------------------------------------------------
  322. SELECT DISTINCT library, symbol
  323. INTO TABLE lib_exp
  324. FROM stlib_exp
  325. UNION
  326. SELECT DISTINCT library, symbol
  327. FROM shlib_exp ;
  328. CREATE TABLE duplicates AS
  329. SELECT DISTINCT symbol
  330. FROM lib_exp
  331. GROUP BY symbol
  332. HAVING COUNT(*) > 1 ;
  333. CREATE TABLE duplicates2 AS
  334. SELECT *
  335. FROM lib_exp
  336. WHERE symbol IN (
  337. SELECT symbol
  338. FROM duplicates
  339. ) ;
  340. SELECT DISTINCT library, symbol
  341. INTO TABLE lib_imp
  342. FROM stlib_imp
  343. UNION
  344. SELECT DISTINCT library, symbol
  345. FROM shlib_imp ;
  346. CREATE TABLE imports AS
  347. SELECT a.library, a.symbol
  348. FROM lib_imp a
  349. WHERE NOT EXISTS (
  350. SELECT b.library, b.symbol
  351. FROM lib_exp b
  352. WHERE b.symbol = a.symbol
  353. AND b.library = a.library
  354. ) ;
  355. CREATE TABLE defined AS
  356. SELECT DISTINCT symbol
  357. FROM lib_exp ;
  358. CREATE TABLE used AS
  359. SELECT DISTINCT symbol
  360. FROM imports ;
  361. CREATE TABLE undefined AS
  362. SELECT symbol
  363. FROM used u
  364. WHERE NOT EXISTS (
  365. SELECT *
  366. FROM defined d
  367. WHERE d.symbol = u.symbol
  368. ) ;
  369. SELECT symbol
  370. INTO TABLE undefined_1
  371. FROM undefined
  372. EXCEPT
  373. SELECT b.symbol
  374. FROM undefined a, libs b
  375. WHERE a.symbol = b.symbol ;
  376. CREATE TABLE undefined_2 AS
  377. SELECT i.symbol, i.object, i.library
  378. FROM stlib_imp i, undefined_1 u
  379. WHERE i.symbol = u.symbol ;
  380. CREATE TABLE depends AS
  381. SELECT i.library AS im_lib,
  382. i.symbol AS symbol,
  383. e.library AS ex_lib
  384. FROM imports i, lib_exp e
  385. WHERE i.symbol = e.symbol ;
  386. CREATE TABLE lib_deps AS
  387. SELECT DISTINCT im_lib, ex_lib
  388. FROM depends
  389. WHERE im_lib <> ex_lib ;
  390. CREATE TABLE lib_deps_1 AS
  391. SELECT a.im_lib,
  392. a.ex_lib AS in_lib,
  393. b.ex_lib
  394. FROM lib_deps a, lib_deps b
  395. WHERE a.ex_lib = b.im_lib ;
  396. CREATE TABLE lib_deps_2 AS
  397. SELECT a.im_lib,
  398. a.in_lib AS in1_lib,
  399. a.ex_lib AS in2_lib,
  400. b.ex_lib
  401. FROM lib_deps_1 a, lib_deps b
  402. WHERE a.ex_lib = b.im_lib
  403. AND a.im_lib <> a.ex_lib ;
  404. SELECT im_lib, ex_lib
  405. INTO TABLE lib_deps_trans
  406. FROM lib_deps
  407. UNION
  408. SELECT im_lib, ex_lib
  409. FROM lib_deps_1
  410. UNION
  411. SELECT im_lib, ex_lib
  412. FROM lib_deps_2 ;
  413. CREATE TABLE prog_libs AS
  414. SELECT DISTINCT a.program, b.library
  415. FROM prog_imp a, lib_exp b
  416. WHERE a.symbol = b.symbol ;
  417. SELECT DISTINCT a.symbol
  418. INTO TABLE libc
  419. FROM prog_imp a, libs b
  420. WHERE a.symbol = b.symbol
  421. AND b.library = 'libc.so.6'
  422. UNION
  423. SELECT DISTINCT a.symbol
  424. FROM imports a, libs b
  425. WHERE a.symbol = b.symbol
  426. AND b.library = 'libc.so.6' ;
  427. SELECT symbol
  428. INTO nonansi
  429. FROM libc
  430. WHERE symbol !~ '_.*'
  431. EXCEPT
  432. SELECT symbol
  433. FROM ansi ;
  434. CREATE TABLE nonansi_progs AS
  435. SELECT a.program, a.symbol
  436. FROM prog_imp a, nonansi b
  437. WHERE a.symbol = b.symbol ;
  438. CREATE TABLE nonansi_libs AS
  439. SELECT a.library, a.symbol
  440. FROM imports a, nonansi b
  441. WHERE a.symbol = b.symbol ;
  442. CREATE TABLE nonansi_prog_counts AS
  443. SELECT symbol, COUNT(*)
  444. FROM nonansi_progs
  445. GROUP BY symbol ;
  446. CREATE TABLE nonansi_lib_counts AS
  447. SELECT symbol, COUNT(*)
  448. FROM nonansi_libs
  449. GROUP BY symbol ;
  450. SELECT symbol
  451. INTO TABLE nonansi_counts
  452. FROM nonansi_prog_counts
  453. UNION
  454. SELECT symbol
  455. FROM nonansi_lib_counts ;
  456. ALTER TABLE nonansi_counts
  457. ADD COLUMN progs INTEGER ;
  458. ALTER TABLE nonansi_counts
  459. ADD COLUMN libs INTEGER ;
  460. UPDATE nonansi_counts
  461. SET progs = 0, libs = 0 ;
  462. UPDATE nonansi_counts
  463. SET progs = b.count
  464. FROM nonansi_prog_counts b
  465. WHERE nonansi_counts.symbol = b.symbol ;
  466. UPDATE nonansi_counts
  467. SET libs = c.count
  468. FROM nonansi_lib_counts c
  469. WHERE nonansi_counts.symbol = c.symbol;
  470. -- SELECT a.symbol, a.program
  471. -- FROM prog_imp a, nonansi_prog_counts b
  472. -- WHERE a.symbol = b.symbol
  473. -- AND a.program NOT LIKE 'bin/%'
  474. -- ORDER BY b.count DESC, b.symbol ;
  475. -- SELECT symbol, library
  476. -- FROM duplicates2
  477. -- ORDER BY symbol ;
  478. -- SELECT a.im_lib, a.ex_lib
  479. -- FROM lib_deps a, lib_deps b
  480. -- WHERE a.ex_lib = b.im_lib
  481. -- AND b.ex_lib = a.im_lib ;
  482. -- SELECT * FROM lib_deps_2
  483. -- WHERE im_lib = ex_lib ;
  484. -- SELECT * FROM lib_deps_1
  485. -- WHERE im_lib = ex_lib ;
  486. -- SELECT im_lib FROM lib_deps_trans
  487. -- WHERE im_lib = ex_lib ;
  488. -- SELECT a.program, a.library
  489. -- FROM ldd a
  490. -- WHERE a.library LIKE 'libgrass_%.so'
  491. -- AND a.library NOT IN (
  492. -- SELECT b.library
  493. -- FROM prog_libs b
  494. -- WHERE b.program = a.program
  495. -- ) ;
  496. -- ----------------------------------------------------------------------
  497. EOF