#!/bin/env ruby ############################################################################ #Word sorter for Pat's dissertation # #More documentation TBD # #Current Usage: # # Scans a file for groups of words and counts the totals # # Accepts the following options: # # --file - the name of the text to sort # # --type - iat or pn # # --bin-file - the name of the bin csv file # # # # The input file is human readable, easy to edit. # # The output file is single-line JSON, use jq to format for reading. # # # #To Do: # ############################################################################ require 'json' require 'pp' require 'csv' require 'optparse' #bin_counter expects: # bin - a hash with strings to search for # test - the document text to search through #This method returns a hash containing the strings and their frequency def bin_counter (bin, text) ret = Hash.new bin.each do |word| ret[word.to_sym] = text.scan(word).count.to_i end return ret end #count_total expects: # bin - a hash created by bin_counter #This method returns a hash with the total count of all words in a bin def count_total (bin) count = 0 bin[:words].each_key do |word| count += bin[:words][word].to_i end return count end #write_output expects: # output - a hash containing all of our output #This method converts the output hash to JSON and writes it to "output.json" def write_output_json (output, filename) outfile = File.open(filename,'w') outfile.write(output.to_json) outfile.close end def write_output_csv (output, filename) CSV.open(filename, 'wb') do |csv| csv << ["bin", "words", "total"] output.each_key do |key| line = [] line.push(key) output[key].each_key do |sub_key| line.push(output[key][sub_key]) end csv << line end end end #strip_text expects: # text - the text we're working on # start - the starting string to search for # fin - the ending string to search for #This method strips out all test before "start" and after "fin" def strip_text (text, start, fin) text.lines(start,chomp: true)[1].lines(fin,chomp: true)[0] end #split_text expects: # text - the text we're working on # start - the starting string to search for # fin the ending string to search for # This method returns everything between start and fin and handles multiple sections # Example: # For the following text: # > asdfasdf # > AAAAA # > testing # > abcdefg # > BBBBB # > nothing # > something # > AAAAA # > moo said the cow. # > cluck said the chicken. # > BBBBB # > bark said the dog # Running split_text(text, 'AAAAA', 'BBBBB') would return: # > testing # > abcdefg # > moo said the cow. # > cluck said the chicken. def split_text (text, start, fin) split1 = text.lines(start, chomp: true)[1..-1] split2 = [] ret = "" split1.each do |section| split2 += section.lines(fin, chomp: true) end split2.each do |section| ret += section end return ret end #process_file expects: # file_name - the name of the file to process # binfile - the name of the bin file (csv) to use # type - which type of file are we processing, must be 'pn' or 'iat' # #This method is the meat and potatos. Preforms the text stripping, word counting, and creates output files. def process_file (file_name, binfile, type) csv = CSV.read(binfile) text = File.read(file_name) text = strip_text(text, 'PLOVEINTAKE', 'PLOVECLOSING') if type == 'iat' text = split_text(text, 'Narrative:', 'Signatures:') if type == 'pn' output = Hash.new #Creating the output storage object bins = Hash.new #This hash stores the bins outfile = file_name outfile.slice!('.txt') #puts outfile csv.each { |bin| bins[bin[0]] = bin[1..].compact } #turn the csv array into a hash, remove nils bins.each_key do |bin_number| key = bin_number.to_sym output[key] = Hash.new output[key][:words] = bin_counter(bins[bin_number], text) output[key][:total] = count_total(output[key]) end write_output_json(output,outfile + '-out.json') write_output_csv(output,outfile + '-out.csv') end #process_dir expects: # dir_name - a direcroty containing text files to process # binfile - the name of the bin file # type - which type of file are we processing, must be 'pn' or 'iat' # #This method will process all .txt files in the supplied directory def process_dir(dir_name, binfile, type) Dir.glob(dir_name + '*.txt') do |file_name| puts file_name process_file(file_name, binfile, type) end end options = Hash.new OptionParser.new do |opts| opts.banner = 'sorter.rb --options' opts.on("-f", "--file file", "Name of the file to process") do |file| options[:file] = file end opts.on("-t", "--type type", "Type of file. Must be \"iat\" or \"pn\"") do |type| options[:type] = type end opts.on("-b", "--bin-file binfile", "Name of the bin file") do |binfile| options[:binfile] = binfile end opts.on("-d", "--directory dir", "Directory containing text files to process") do |dir| options[:dir] = dir end end.parse! if options[:file] && options[:dir] puts "Invalid options, you must either a file or a directoy of files." elsif options[:file] process_file(options[:file], options[:binfile], options[:type]) elsif options[:dir] process_dir(options[:dir], options[:binfile], options[:type]) end