Poor mans data_collector forever

Dont let data collector info roll over.

Here is how you can save it into a vertica database before its rolled over and deleted.

# create a dc schema for all dc tables 
vsql -c "create schema dc;"
mkdir temp

# in catalog directory
cd DataCollector

# load a copy of the DC Schema
for file in CREATE_*.sql ; do vsql -a -f $file; done

## to load data, watch for deleted files and load them into vertica

while read -r event_filename_ts;  do 
	#echo --${event_filename_ts}; 
	event=$(echo $event_filename_ts | sed "s/\(.*\) \(.*\)_\(.*\).log/\1/g")
	filename=$(echo $event_filename_ts | sed "s/\(.*\) \(.*\)_\(.*\).log/\2/g")
	ts=$(echo $event_filename_ts | sed "s/\(.*\) \(.*\)_\(.*\).log/\3/g")
	#echo --${event}
	#echo --{$filename}
	#echo --${ts}
	if [ "${event}" = "CREATE" ]; then
	# create a temporary link to the file
	ln ${filename}_${ts}.log temp/${filename}_${ts}.ln
	elif [ "${event}" = "DELETE" -a -e temp/${filename}_${ts}.ln ] ; then
	# load the file and dispose of it
	sed "s/${filename}_\*\.log/temp\/${filename}_${ts}.ln/g" COPY_${filename}_TABLE.sql | vsql
	rm -rf temp/${filename}_${ts}.ln
	fi
done < <(inotifywait -m  -e "CREATE,DELETE" --format "%e %f"  .  )  


## clean up when done
rm -rf temp

remember

1. do it on all nodes

2. deamonize this script and run it as a service