Your own indexing tool complemented by your own name search tool

ericmarceau · 6 September 2024 02:50

The search tool that is incorporated into Caja does not meet my needs.

I have weird ways of looking for things, using weird patterns. So, I created my own full-system indexing tool and my own search tool. I believe these have sufficient commentary to be self-explanatory. Enjoy!

Session log for "UTIL__SysIndex__Build.sh":

root:~# UTIL__SysIndex__Build.sh
 Rebuild partition index for partition DB001_F1 ? [y|N] => 

 Rebuild partition index for partition DB001_F2 ? [y|N] => y

 Rebuild partition index for partition DB001_F3 ? [y|N] => 

 Rebuild partition index for partition DB001_F4 ? [y|N] => y

 Rebuild partition index for partition DB001_F5 ? [y|N] => y

 Rebuild partition index for partition DB001_F6 ? [y|N] => 

 Rebuild partition index for partition DB001_F7 ? [y|N] => y

 Rebuild partition index for partition DB001_F8 ? [y|N] => 

 Worklist:   DB001_F2 DB001_F4 DB001_F5 DB001_F7 ...


 Evaluating for DB001_F2 ...
  DOING directories:
   find -H /DB001_F2 -xdev \(   -type d \) \( ! -name '*_files' \) -print >/DB001_F2/LO_Index/DB001_F2.d.INDEX.txt ...
   -rw-r--r-- 1 root root 680516 Sep  5 22:42 /DB001_F2/LO_Index/DB001_F2.d.INDEX.txt
  DOING files:
   find -H /DB001_F2 -xdev \( ! -type d \) -print | grep -v '_files/' >/DB001_F2/LO_Index/DB001_F2.f.INDEX.txt ...
   -rw-r--r-- 1 root root 12777530 Sep  5 22:42 /DB001_F2/LO_Index/DB001_F2.f.INDEX.txt

 Evaluating for DB001_F4 ...
  DOING directories:
   find -H /DB001_F4 -xdev \(   -type d \) \( ! -name '*_files' \) -print >/DB001_F2/LO_Index/DB001_F4.d.INDEX.txt ...
   -rw-r--r-- 1 root root 1275509 Sep  5 22:42 /DB001_F2/LO_Index/DB001_F4.d.INDEX.txt
  DOING files:
   find -H /DB001_F4 -xdev \( ! -type d \) -print | grep -v '_files/' >/DB001_F2/LO_Index/DB001_F4.f.INDEX.txt ...
grep: (standard input): binary file matches
   -rw-r--r-- 1 root root 21986968 Sep  5 22:42 /DB001_F2/LO_Index/DB001_F4.f.INDEX.txt

 Evaluating for DB001_F5 ...
  DOING directories:
   find -H /DB001_F5 -xdev \(   -type d \) \( ! -name '*_files' \) -print >/DB001_F2/LO_Index/DB001_F5.d.INDEX.txt ...
   -rw-r--r-- 1 root root 1881383 Sep  5 22:43 /DB001_F2/LO_Index/DB001_F5.d.INDEX.txt
  DOING files:
   find -H /DB001_F5 -xdev \( ! -type d \) -print | grep -v '_files/' >/DB001_F2/LO_Index/DB001_F5.f.INDEX.txt ...
grep: (standard input): binary file matches
   -rw-r--r-- 1 root root 15052959 Sep  5 22:43 /DB001_F2/LO_Index/DB001_F5.f.INDEX.txt

 Evaluating for DB001_F7 ...
  DOING directories:
   find -H /DB001_F7 -xdev \(   -type d \) \( ! -name '*_files' \) -print >/DB001_F2/LO_Index/DB001_F7.d.INDEX.txt ...
   -rw-r--r-- 1 root root 54876 Sep  5 22:43 /DB001_F2/LO_Index/DB001_F7.d.INDEX.txt
  DOING files:
   find -H /DB001_F7 -xdev \( ! -type d \) -print | grep -v '_files/' >/DB001_F2/LO_Index/DB001_F7.f.INDEX.txt ...
   -rw-r--r-- 1 root root 987379 Sep  5 22:43 /DB001_F2/LO_Index/DB001_F7.f.INDEX.txt

 Creating combined global INDEX (directories) ...
   removed '/DB001_F2/LO_Index/INDEX.allDrives.d.txt'
   Old directory index purged ...
   New global directory index created ...
   -rw-r--r-- 1 root root 7480933 Sep  5 22:43 /DB001_F2/LO_Index/INDEX.allDrives.d.txt

 Creating combined global INDEX (files) ...
   removed '/DB001_F2/LO_Index/INDEX.allDrives.f.txt'
   Old file index purged ...
   New global file index created ...
   -rw-r--r-- 1 root root 104299952 Sep  5 22:43 /DB001_F2/LO_Index/INDEX.allDrives.f.txt

 DONE!

root:~#

UTIL__SysIndex__Build.sh:

#!/bin/sh

#23456789+123456789+123456789+123456789+123456789+123456789+123456789+123456789+123456789+123456789+
####################################################################################################
###
###	$Id: UTIL__SysIndex__Build.sh,v 1.1 2021/11/24 13:27:01 root Exp root $
###
###	Script to create simple index list for each of directories and files for each partition on the system's boot disk.
###
####################################################################################################


##################################################################################################
##################################################################################################
doSearchIndex()
{
	if [ ! -d ${procROOT} ]
	then
		echo "\n\t Index storage location '${procROOT}' is missing.  Abandoning process.  Bye!\n" ; exit 1
	fi

	PartitionIndexD="${procROOT}/${PartitionLabel}.d.INDEX.txt"
	PartitionIndexF="${procROOT}/${PartitionLabel}.f.INDEX.txt"
	rm -f ${PartitionIndexD}
	rm -f ${PartitionIndexF}

	if [ ${doHtml} -eq 1 ]
	then
		## Logic including contents of '*.files' directories
		COMd="find -H ${SearchPath} -xdev \(   -type d \) -print >${PartitionIndexD}"
		COMf="find -H ${SearchPath} -xdev \( ! -type d \) -print >${PartitionIndexF}"
	else
		## Logic excluding contents of '*.files' directories
		COMd="find -H ${SearchPath} -xdev \(   -type d \) \( ! -name '*_files' \) -print >${PartitionIndexD}"
		#COMf="find -H ${SearchPath} -xdev \( ! -type d \) \( ! -regex '*_files/*' \) -print >${PartitionIndexF}"
		COMf="find -H ${SearchPath} -xdev \( ! -type d \) -print | grep -v '_files/' >${PartitionIndexF}"
	fi

	echo "  DOING directories:\n   ${COMd} ..."
	eval ${COMd}

	mv	${PartitionIndexD}    ${PartitionIndexD}2
	sort	${PartitionIndexD}2  >${PartitionIndexD}
	rm -f	${PartitionIndexD}2
	ls -l ${PartitionIndexD} 2>&1 | awk '{ printf("   %s\n", $0 ) ; }'

	echo "  DOING files:\n   ${COMf} ..."
	eval ${COMf}

	mv	${PartitionIndexF}    ${PartitionIndexF}2
	sort	${PartitionIndexF}2  | grep '^/' >${PartitionIndexF}
	rm -f	${PartitionIndexF}2
	ls -l ${PartitionIndexF} 2>&1 | awk '{ printf("   %s\n", $0 ) ; }'
}	#doSearchIndex()


##################################################################################################
##################################################################################################
askDoDrive()
{
	echo "\n Rebuild partition index for partition ${PartitionLabel} ? [y|N] => \c"
	read ans

	if [ -z "${ans}" ]
	then
		ans="N"
	fi

	case ${ans} in
		y* | Y* ) 	doList="${doList} ${PartitionLabel}"
				#doThis="Y"
				;;
		* )		#doThis="N"
				;;
	esac
}	#askDoDrive()


##################################################################################################
##################################################################################################
##################################################################################################

TMP=/tmp/tmp.`basename $0 ".sh" `.$$

doHtml=0
rebuild=0
doList=""

while [ $# -gt 0 ]
do
	case $1 in
		--partition )	doList="${2}" ; echo "\n\n STARTING:  $0 ...\n" ; shift ; shift ;;
		--htmlFILES )	doHtml=1	; shift ;;
		--rebuildALL )	rebuild=1	; shift ;;
		* ) echo "\n\t Invalid parameter used on command line.  Only options allowed: [ --htmlFILES | --rebuildALL ] \n" ; exit 1 ;;
	esac
done

case `hostname` in
	OasisMega1 )
		procROOT="/DB001_F2/LO_Index"
		PartitionLabel=DB001_F1
		indexMode=2
		;;
	OasisMega2 )
		procROOT="/LO_Index"
		PartitionLabel=DB002_F1
		SearchPath=/
		indexMode=1
		;;
#	OasisMidi )
#		procROOT="/LO_Index"
#		PartitionLabel=DB003_F1
#		SearchPath=/
#		indexMode=1
#		;;
#	OasisMini )
#		procROOT="/LO_Index"
#		PartitionLabel=DB004_F1
#		SearchPath=/
#		indexMode=1
#		;;
	* ) echo "\n\t Logic has not been defined to preform indexing for this host.\n Bye!\n" ; exit 1 ;;
esac

cd /
count=`ls -d DB00?_F? 2>>/dev/null | wc -l | awk '{ print $1 }' `

if [ ${rebuild} -eq 1 ]
then
	rm -f ${TMP}.listDirIndexes 2>>/dev/null ; ls ${procROOT}/DB00?_F?.d.INDEX.txt   >${TMP}.listDirIndexes 2>>/dev/null
	rm -f ${TMP}.listFilIndexes 2>>/dev/null ; ls ${procROOT}/DB00?_F?.f.INDEX.txt   >${TMP}.listFilIndexes 2>>/dev/null
fi

if [ -s ${TMP}.listDirIndexes ] ; then  rm -fv `cat ${TMP}.listDirIndexes ` ; fi
if [ -f ${TMP}.listFilIndexes ] ; then  rm -fv `cat ${TMP}.listFilIndexes ` ; fi

if [ -z "${doList}" ]
then
	for PartitionLabel in DB00?_F?
	do
		askDoDrive
	done
fi

echo "\n Worklist:  ${doList} ...\n"

for PartitionLabel in `echo ${doList} `
do
	SearchPath=/${PartitionLabel}

	case ${PartitionLabel} in
		#DB001_F1 | DB002_F1 | DB003_F1 | DB004_F1 )
		DB001_F1 | DB002_F1 )
			SearchPath=/
			#askDoDrive

			#if [ "${doThis}" = "Y" ]
			#then
				echo "\n Evaluating for ${PartitionLabel} ..."

				doSearchIndex
			#fi
			;;
		* )	
			#askDoDrive

			#if [ "${doThis}" = "Y" ]
			#then
			#	echo "\n\t Suppressed indexing of ${PartitionLabel} as chosen ..."
			#else
				test1=`df -h / | grep '/dev' | awk '{ print $1 }' `
				test2=`df -h ${SearchPath} | grep '/dev' | awk '{ print $1 }' `

				if [ "${test1}" = "${test2}" ]
				then
					echo "\n\t Suppressed indexing of ${PartitionLabel} as same partition as root ..."
				else
					echo "\n Evaluating for ${PartitionLabel} ..."

					doSearchIndex
				fi
			#fi
			;;
	esac
done

echo ""
rm -f ${TMP}.listDirIndexes 2>>/dev/null ; ls ${procROOT}/DB00?_F?.d.INDEX.txt   >${TMP}.listDirIndexes 2>>/dev/null

if [ -s ${TMP}.listDirIndexes ]
then
	echo " Creating combined global INDEX (directories) ..."
	rm -fv ${procROOT}/INDEX.allDrives.d.txt 2>&1 | awk '{ printf("   %s\n", $0 ) ; }'
	echo "   Old directory index purged ..."
	cat ${procROOT}/DB00?_F?.d.INDEX.txt   >${procROOT}/INDEX.allDrives.d.txt
	echo "   New global directory index created ..."
	ls -l ${procROOT}/INDEX.allDrives.d.txt 2>&1 | awk '{ printf("   %s\n", $0 ) ; }'
else
	echo " ERROR:  No directory index files for individual partitions.  Please investigate.\n" ; exit 1
fi

echo ""
rm -f ${TMP}.listFilIndexes 2>>/dev/null ; ls ${procROOT}/DB00?_F?.f.INDEX.txt   >${TMP}.listFilIndexes 2>>/dev/null

if [ -s ${TMP}.listFilIndexes ]
then
	echo " Creating combined global INDEX (files) ..."
	rm -fv ${procROOT}/INDEX.allDrives.f.txt 2>&1 | awk '{ printf("   %s\n", $0 ) ; }'
	echo "   Old file index purged ..."
	cat ${procROOT}/DB00?_F?.f.INDEX.txt   >${procROOT}/INDEX.allDrives.f.txt
	echo "   New global file index created ..."
	ls -l ${procROOT}/INDEX.allDrives.f.txt 2>&1 | awk '{ printf("   %s\n", $0 ) ; }'
else
	echo " ERROR:  No file index files for individual partitions.  Please investigate.\n" ; exit 1
fi

echo "\n DONE!\n"

rm -f ${TMP}.listDirIndexes ${TMP}.listFilIndexes

exit 0
exit 0
exit 0


###############################################################################
###############################################################################
###############################################################################

Session log for "UTIL__SearchIndexes.sh":

root:~#  ./UTIL__SearchIndexes.sh --single --notype Android Cookbook

	 [Aa][Nn][Dd][Rr][Oo][Ii][Dd]
	 [Cc][Oo][Oo][Kk][Bb][Oo][Oo][Kk]

25M	/DB001_F5/TOPIC__Programming/PROGRAMMING__Android/Android Cookbook, 2nd Edition.pdf
11M	/DB001_F5/TOPIC__Programming/PROGRAMMING__Android/Android Cookbook.pdf
root:~#

UTIL__SearchIndexes.sh:

#!/bin/bash

index=${index:-/DB001_F2/LO_Index}

usage()
{
	printf "\n\t usage: $(basename "$0") [ --single ] [ --notype ] [-={ftype} ] [ --pattern ] {string} \n\n" ; exit 1
}

if [ $# -eq 0 ]
then
	usage
fi

doFiles=1
doSingle=0
ftype=pdf
patternOnly=0

while [ $# -gt 0 ]
do
	case $1 in
		--fils ) doFiles=1 ; shift ;;
		--dirs ) doFiles=0 ; ftype="" ; shift ;;
		--basic ) doSingle=1 ; ftype="" ; shift ;;
		--single ) doSingle=1 ; shift ;;
		--notype ) ftype="" ; shift ;;
		--pattern ) patternOnly=1 ; shift ; break ;;
		-=* )	ftype=$(echo "$1" | cut -c3- ) ; shift ;;
		--* ) printf "\n\t Invalid option specified.\n" ; usage ; exit 1 ;;
		* )	break ;;
	esac
done
#printf "${ftype}\n"

cd ${index}

if [ ${doFiles} -eq 1 ]
then
	INDEX="INDEX.allDrives.f.txt"
else
	INDEX="INDEX.allDrives.d.txt"
fi

makePatternMatch()
{
	echo "${charList}" | awk 'BEGIN{
		regExp="" ;
	}
	{
		if( $0 != "" ){
			for( i=1 ; i <= NF ; i++ ){
				if( $i ~ /[[:alpha:]]/ ){
					regExp=sprintf("%s[%s%s]", regExp, toupper($i), tolower($i) ) ;
				}else{
					regExp=sprintf("%s%s", regExp, $i ) ;
				} ;
			} ;
		} ;
	}END{
		printf("%s\n", regExp ) ;
	}'
}

explodeString()
{
	#charList=$(echo "${pattern}" | sed 's+[[:alpha:]]*+&\ +g' )
	#charList=$(echo "${pattern}" | sed 's+[[:alpha:]]+&\ +g' )
	charList=$(echo "${pattern}" | sed 's+.+&\ +g' )
	echo "${charList}" >&2
}


method1()
{
	for pattern in $@
	do
		explodeString
		makePatternMatch
	done
}
#method1 $@
### Output - Method 1
#a b * c 1 
#[Aa][Bb]*[Cc]1
#d 2 e ? f 
#[Dd]2[Ee]?[Ff]


method2()
{
	echo "$@" |
	awk '{
		if( $0 != "" ){
			for ( j=1 ; j<= NF ; j++ ){
				n=split( $j , arr , "" ) ;	###  Split word-string into array
				for ( i=1 ; i<=(n) ; i++ ){
					if ( arr[i] ~ /[[:alpha:]]/ ){
						printf("[%s%s]", toupper(arr[i]), tolower(arr[i]) ) ;
					}else{
						printf("%s", arr[i] ) ;
					} ;
				} ;
				printf("\n") ;
			} ;
		} ;
	}'
}
#method2 $@
### Output - Method 2
#[Aa][Bb]*[Cc]1
#[Dd]2[Ee]?[Ff]


method3()
{
	###	$@ at this level is limited to scope of function, i.e. variables passed at function call.
	for pattern in $@
	do
		echo "${pattern}" | sed 's/[[:alpha:]]/[\u&\l&]/g'
	done
}

###	$@ at this level is visible only to top-level code, not code wrapped in functions.
### Output - Method 3
#[Aa][Bb]*[Cc]1
#[Dd]2[Ee]?[Ff]


if [ ${patternOnly} -eq 1 ]
then
	for strs in $@
	do
		method3 ${strs}
	done | awk 'BEGIN{ printf("\n") ; }{ printf("\t %s\n", $0 ) ; }END{ printf("\n") ; }'
	exit
else
	patterns=()
	patterns[0]=""

	i=0
	for strs in $@
	do
		i=$((i+=1))
		patterns[${i}]=$(method3 ${strs} )
	done
fi

doFtype()
{
	rm -f UTIL__SearchIndexes.exceptions
	{
	if [ -z "${ftype}" ]
	then
		#printf "here\n" >&2
		cat
	else
		ftype=$(method3 ${ftype} )
		grep \."${ftype}"\$
	fi
	} | xargs -I '{}' du -sh '{}' 2>UTIL__SearchIndexes.exceptions
}

case $# in
	1 )
		{
			printf "\n\t ${patterns[1]}\n\n"
		} >&2
		grep "${patterns[1]}" INDEX.allDrives.f.txt |
			doFtype
			#		grep \."${ftype}"\$
		;;
	2 )
		{
			printf "\n\t ${patterns[1]}\n"
			printf "\t ${patterns[2]}\n\n"
		} >&2

		{	if [ ${doSingle} -eq 1 ]
			then
				grep "${patterns[1]} ${patterns[2]}" INDEX.allDrives.f.txt
			else
				grep "${patterns[1]}" INDEX.allDrives.f.txt	|
				grep "${patterns[2]}"
			fi
		} | doFtype
			#		grep \."${ftype}"\$
		;;
	3 )
		{
			printf "\n\t ${patterns[1]}\n"
			printf "\t ${patterns[2]}\n"
			printf "\t ${patterns[3]}\n\n"
		} >&2

		{	if [ ${doSingle} -eq 1 ]
			then
				grep "${patterns[1]} ${patterns[2]} ${patterns[3]}" INDEX.allDrives.f.txt
			else
				grep "${patterns[1]}" INDEX.allDrives.f.txt	|
				grep "${patterns[2]}"				|
				grep "${patterns[3]}"
			fi
		} | doFtype
			#		grep \."${ftype}"\$
		;;
	4 )
		{
			printf "\n\t ${patterns[1]}\n"
			printf "\t ${patterns[2]}\n"
			printf "\t ${patterns[3]}\n"
			printf "\t ${patterns[4]}\n\n"
		} >&2

		{	if [ ${doSingle} -eq 1 ]
			then
				grep "${patterns[1]} ${patterns[2]} ${patterns[3]} ${patterns[4]}" INDEX.allDrives.f.txt
			else
				grep "${patterns[1]}" INDEX.allDrives.f.txt	|
				grep "${patterns[2]}"				|
				grep "${patterns[3]}"				|
				grep "${patterns[4]}"
			fi
		} | doFtype
			#		grep \."${ftype}"\$
		;;
	5 )
		{
			printf "\n\t ${patterns[1]}\n"
			printf "\t ${patterns[2]}\n"
			printf "\t ${patterns[3]}\n"
			printf "\t ${patterns[4]}\n"
			printf "\t ${patterns[5]}\n\n"
		} >&2

		{	if [ ${doSingle} -eq 1 ]
			then
				grep "${patterns[1]} ${patterns[2]} ${patterns[3]} ${patterns[4]} ${patterns[5]}" INDEX.allDrives.f.txt
			else
				grep "${patterns[1]}" INDEX.allDrives.f.txt	|
				grep "${patterns[2]}"				|
				grep "${patterns[3]}"				|
				grep "${patterns[4]}"				|
				grep "${patterns[5]}"
			fi
		} | doFtype
			#		grep \."${ftype}"\$
		;;
	6 )
		{
			printf "\n\t ${patterns[1]}\n"
			printf "\t ${patterns[2]}\n"
			printf "\t ${patterns[3]}\n"
			printf "\t ${patterns[4]}\n"
			printf "\t ${patterns[5]}\n"
			printf "\t ${patterns[6]}\n\n"
		} >&2

		{	if [ ${doSingle} -eq 1 ]
			then
				grep "${patterns[1]} ${patterns[2]} ${patterns[3]} ${patterns[4]} ${patterns[5]} ${patterns[6]}" INDEX.allDrives.f.txt
			else
				grep "${patterns[1]}" INDEX.allDrives.f.txt	|
				grep "${patterns[2]}"				|
				grep "${patterns[3]}"				|
				grep "${patterns[4]}"				|
				grep "${patterns[5]}"				|
				grep "${patterns[6]}"
			fi
		} | doFtype
			#		grep \."${ftype}"\$
		;;
	7 )
		{
			printf "\n\t ${patterns[1]}\n"
			printf "\t ${patterns[2]}\n"
			printf "\t ${patterns[3]}\n"
			printf "\t ${patterns[4]}\n"
			printf "\t ${patterns[5]}\n"
			printf "\t ${patterns[6]}\n"
			printf "\t ${patterns[7]}\n\n"
		} >&2

		{	if [ ${doSingle} -eq 1 ]
			then
				grep "${patterns[1]} ${patterns[2]} ${patterns[3]} ${patterns[4]} ${patterns[5]} ${patterns[6]} ${patterns[7]}" INDEX.allDrives.f.txt
			else
				grep "${patterns[1]}" INDEX.allDrives.f.txt	|
				grep "${patterns[2]}"				|
				grep "${patterns[3]}"				|
				grep "${patterns[4]}"				|
				grep "${patterns[5]}"				|
				grep "${patterns[6]}"				|
				grep "${patterns[7]}"
			fi
		} | doFtype
			#		grep \."${ftype}"\$
		;;
	8 )
		{
			printf "\n\t ${patterns[1]}\n"
			printf "\t ${patterns[2]}\n"
			printf "\t ${patterns[3]}\n"
			printf "\t ${patterns[4]}\n"
			printf "\t ${patterns[5]}\n"
			printf "\t ${patterns[6]}\n"
			printf "\t ${patterns[7]}\n"
			printf "\t ${patterns[8]}\n\n"
		} >&2

		{	if [ ${doSingle} -eq 1 ]
			then
				grep "${patterns[1]} ${patterns[2]} ${patterns[3]} ${patterns[4]} ${patterns[5]} ${patterns[6]} ${patterns[7]} ${patterns[8]}" INDEX.allDrives.f.txt
			else
				grep "${patterns[1]}" INDEX.allDrives.f.txt	|
				grep "${patterns[2]}"				|
				grep "${patterns[3]}"				|
				grep "${patterns[4]}"				|
				grep "${patterns[5]}"				|
				grep "${patterns[6]}"				|
				grep "${patterns[7]}"				|
				grep "${patterns[8]}"
			fi
		} | doFtype
			#		grep \."${ftype}"\$
		;;
esac

pavlos_kairis · 6 September 2024 11:21

have you tried mlocate updatedb ?

https://linux.die.net/man/8/updatedb

ericmarceau · 6 September 2024 16:37

Thank you, @pavlos_kairis. I looked at that, long ago, and walked away. I prefer my own script where I feel I have a better sense of control over what I am looking for.