generate.py

来自「这是一个用与生成数据库测试文件的程序对于网站的测试有很大的帮助」· Python 代码 · 共 1,725 行 · 第 1/5 页

    field_names.append(field_dict['name'])

  if (field_dict.get('type','') not in ['freq','date','phone','ident']):
    print 'Error: Illegal or no field type given for field "%s": %s' % \
          (field_dict['name'], field_dict.get('type',''))
    raise Exception

  if (field_dict.get('char_range','') not in ['alpha', 'alphanum','digit']):
    print 'Error: Illegal or no random character range given for ' + \
          'field "%s": %s' % (field_dict['name'], \
                              field_dict.get('char_range',''))
    raise Exception

  if (field_dict['type'] == 'freq'):
    if (not field_dict.has_key('freq_file')):
      print 'Error: Field of type "freq" has no file name given'
      raise Exception

  elif (field_dict['type'] == 'date'):
    if (not (field_dict.has_key('start_date') and \
             field_dict.has_key('end_date'))):
      print 'Error: Field of type "date" has no start and/or end date given'
      raise Exception

    else:  # Process start and end date
      start_date = field_dict['start_date']
      end_date =   field_dict['end_date']

      start_epoch = date_to_epoch(start_date[0], start_date[1], start_date[2])
      end_epoch =   date_to_epoch(end_date[0], end_date[1], end_date[2])
      field_dict['start_epoch'] = start_epoch
      field_dict['end_epoch'] =   end_epoch
      field_list[i] = field_dict

  elif (field_dict['type'] == 'phone'):
    if (not (field_dict.has_key('area_codes') and \
             field_dict.has_key('num_digits'))):
      print 'Error: Field of type "phone" has no area codes and/or number ' + \
            'of digits given'
      raise Exception

    else:  # Process area codes and number of digits
      if (isinstance(field_dict['area_codes'],str)):  # Only one area code
        field_dict['area_codes'] = [field_dict['area_codes']]  # Make it a list
      if (not isinstance(field_dict['area_codes'],list)):
        print 'Error: Area codes given are not a string or a list: %s' % \
              (str(field_dict['area_codes']))
        raise Exception

      if (not isinstance(field_dict['num_digits'],int)):
        print 'Error: Number of digits given is not an integer: %s (%s)' % \
              (str(field_dict['num_digits']), type(field_dict['num_digits']))
        raise Exception

      field_list[i] = field_dict

  elif (field_dict['type'] == 'ident'):
    if (not (field_dict.has_key('start_id') and \
             field_dict.has_key('end_id'))):
      print 'Error: Field of type "iden" has no start and/or end ' + \
            'identification number given'
      raise Exception

  # Check all the probabilities for this field
  #
  if ('select_prob' not in field_dict):
    field_dict['select_dict'] = 0.0
  elif (field_dict['select_prob'] < 0.0) or (field_dict['select_prob'] > 1.0):
    print 'Error: Illegal value for select probability in dictionary for ' + \
          'field "%s": %f' % (field_dict['name'], field_dict['select_prob'])
  else:
    select_prob_sum += field_dict['select_prob']

  field_prob_sum = 0.0

  for prob in prob_names:
    if (prob not in field_dict):
      field_dict[prob] = 0.0
    elif (field_dict[prob] < 0.0) or (field_dict[prob] > 1.0):
      print 'Error: Illegal value for "%s" probability in dictionary for ' % \
            (prob) + 'field "%s": %f' % (field_dict['name'], field_dict[prob])
      raise Exception
    else:
      field_prob_sum += field_dict[prob]

  if (field_prob_sum > 0.0) and (abs(field_prob_sum - 1.0) > 0.001):
      print 'Error: Sum of probabilities for field "%s" is not 1.0: %f' % \
            (field_dict['name'], field_prob_sum)
      raise Exception

  # Create a list of field probabilities and insert into field dictionary
  #
  prob_list = []
  prob_sum =  0.0

  for prob in prob_names:
    prob_list.append((prob, prob_sum))
    prob_sum += field_dict[prob]

  field_dict['prob_list'] = prob_list
  field_list[i] = field_dict  # Store dictionary back into dictionary list

  i += 1

if (abs(select_prob_sum - 1.0) > 0.001):
  print 'Error: Field select probabilities do not sum to 1.0: %f' % \
        (select_prob_sum)
  raise Exception

# Create list of select probabilities - - - - - - - - - - - - - - - - - - - - -
#
select_prob_list = []
prob_sum =         0.0

for field_dict in field_list:
  select_prob_list.append((field_dict, prob_sum))
  prob_sum += field_dict['select_prob']

# -----------------------------------------------------------------------------
# Create a distribution for the number of duplicates for an original record
#
num_dup =  1
prob_sum = 0.0
prob_dist_list = [(num_dup, prob_sum)]

if (prob_distribution == 'uni'):  # Uniform distribution of duplicates - - - -

  uniform_val = 1.0 / float(max_num_dups)

  for i in range(max_num_dups-1):
    num_dup += 1
    prob_dist_list.append((num_dup, uniform_val+prob_dist_list[-1][1]))

elif (prob_distribution == 'poi'):  # Poisson distribution of duplicates - - -

  def fac(n):  # Factorial of an integer number (recursive calculation)
    if (n > 1.0):
      return n*fac(n - 1.0)
    else:
      return 1.0

  poisson_num = []  # A list of poisson numbers
  poisson_sum = 0.0  # The sum of all poisson number

  # The mean (lambda) for the poisson numbers
  #
  mean = 1.0 + (float(num_dup_records) / float(num_org_records))

  for i in range(max_num_dups):
    poisson_num.append((math.exp(-mean) * (mean ** i)) / fac(i))
    poisson_sum += poisson_num[-1]

  for i in range(max_num_dups):  # Scale so they sum up to 1.0
    poisson_num[i] = poisson_num[i] / poisson_sum

  for i in range(max_num_dups-1):
    num_dup += 1
    prob_dist_list.append((num_dup, poisson_num[i]+prob_dist_list[-1][1]))

elif (prob_distribution == 'zip'):  # Zipf distribution of duplicates - - - - -
  zipf_theta = 0.5

  denom = 0.0
  for i in range(num_org_records):
    denom += (1.0 / (i+1) ** (1.0 - zipf_theta))

  zipf_c = 1.0 / denom
  zipf_num = []  # A list of Zipf numbers
  zipf_sum = 0.0  # The sum of all Zipf number

  for i in range(max_num_dups):
    zipf_num.append(zipf_c / ((i+1) ** (1.0 - zipf_theta)))
    zipf_sum += zipf_num[-1]

  for i in range(max_num_dups):  # Scale so they sum up to 1.0
    zipf_num[i] = zipf_num[i] / zipf_sum

  for i in range(max_num_dups-1):
    num_dup += 1
    prob_dist_list.append((num_dup, zipf_num[i]+prob_dist_list[-1][1]))

print
print 'Create %i original and %i duplicate records' % \
      (num_org_records, num_dup_records)
print '  Distribution of number of duplicates (maximal %i duplicates):' % \
      (max_num_dups)
print '  %s' % (prob_dist_list)

# -----------------------------------------------------------------------------
# Load frequency files and misspellings dictionaries
#
print
print 'Step 1: Load and process frequency tables and misspellings dictionaries'

freq_files = {}
freq_files_length = {}

i = 0  # Loop counter
for field_dict in field_list:
  field_name = field_dict['name']

  if (field_dict['type'] == 'freq'):  # Check for 'freq' field type

    file_name = field_dict['freq_file']  # Get the corresponding file name

    if (file_name != None):
      try:
        fin = open(file_name)  # Open file for reading
      except:
        print '  Error: Can not open frequency file %s' % (file_name)
        raise Exception
      value_list = []  # List with all values of the frequency file

      for line in fin:
        line = line.strip()
        line_list = line.split(',')
        if (len(line_list) != 2):
          print '  Error: Illegal format in  frequency file %s: %s' % \
                (file_name, line)
          raise Exception

        line_val =  line_list[0].strip()
        line_freq = int(line_list[1])

        # Append value as many times as given in frequency file
        #
        new_list = [line_val]* line_freq
        value_list += new_list

      random.shuffle(value_list)  # Randomly shuffle the list of values

      freq_files[field_name] = value_list
      freq_files_length[field_name] = len(value_list)

      if (VERBOSE_OUTPUT == True):
        print '  Loaded frequency file for field "%s" from file: %s' % \
              (field_dict['name'], file_name)
        print

    else:
      print '  Error: No file name defined for frequency field "%s"' % \
            (field_dict['name'])
      raise Exception

  if ('misspell_file' in field_dict):  # Load misspellings dictionary file
    misspell_file_name = field_dict['misspell_file']
    field_dict['misspell_dict'] = load_misspellings_dict(misspell_file_name)

    if (VERBOSE_OUTPUT == True):
      print '  Loaded misspellings dictionary for field "%s" from file: "%s' \
            % (field_dict['name'], misspell_file_name)
      print

    field_list[i] = field_dict  # Store dictionary back into dictionary list

  i += 1

# -----------------------------------------------------------------------------
# Create original records
#
print
print 'Step 2: Create original records'
print

org_rec = {}  # Dictionary for original records
all_rec_set = sets.Set()  # Set of all records (without identifier) used for
                          # checking that all records are different
rec_cnt = 0

while (rec_cnt < num_org_records):
  rec_id = 'rec-%i-org' % (rec_cnt)  # The records identifier

  rec_dict = {'rec_id':rec_id}  # Save record identifier

  # Now randomly create all the fields in a record  - - - - - - - - - - - - - -
  #
  for field_dict in field_list:
    field_name = field_dict['name']

    # Randomly set field values to missing
    #
    if (random.random() <= field_dict['miss_prob']):
      rand_val = missing_value

    elif (field_dict['type'] == 'freq'):  # A frequency file based field
      rand_num = random.randint(0, freq_files_length[field_name]-1)
      rand_val = freq_files[field_name][rand_num]

    elif (field_dict['type'] == 'date'):  # A date field
      rand_num = random.randint(field_dict['start_epoch'], \
                                field_dict['end_epoch']-1)
      rand_date = epoch_to_date(rand_num)
      rand_val = rand_date[2]+rand_date[1]+rand_date[0]  # ISO format: yyyymmdd

    elif (field_dict['type'] == 'phone'):  # A phone number field
      area_code = random.choice(field_dict['area_codes'])
      max_digit = int('9'*field_dict['num_digits'])
      min_digit = int('1'*(int(1+round(field_dict['num_digits']/2.))))
      rand_num = random.randint(min_digit, max_digit)
      rand_val = area_code+' '+str(rand_num).zfill(field_dict['num_digits'])

    elif (field_dict['type'] == 'ident'):  # A identification number field
      rand_num = random.randint(field_dict['start_id'], \
                                field_dict['end_id']-1)
      rand_val = str(rand_num)

    if (rand_val != missing_value):  # Don't save missing values
      rec_dict[field_name] = rand_val

  # Create a string representation which can be used to check for uniqueness
  #
  rec_data = rec_dict.copy()  # Make a copy of the record dictionary
  del(rec_data['rec_id'])     # Remove the record identifier
  rec_list = rec_data.items()
  rec_list.sort()
  rec_str = str(rec_list)

  if (rec_str not in all_rec_set):  # Check if same record already created
    all_rec_set.add(rec_str)
    org_rec[rec_id] = rec_dict  # Insert into original records
    rec_cnt += 1

    # Print original record - - - - - - - - - - - - - - - - - - - - - - - - - -
    #
    if (VERBOSE_OUTPUT == True):
      print '  Original:'
      print '    Record ID         : %-30s' % (rec_dict['rec_id'])
      for field_name in field_names:
        print '    %-18s: %-30s' % (field_name, \
                                    rec_dict.get(field_name, missing_value))
      print

  else:
    if (VERBOSE_OUTPUT == True):
      print '***** Record "%s" already crated' % (rec_str)

# -----------------------------------------------------------------------------
# Create duplicate records
#
print
print 'Step 2: Create duplicate records'
print

dup_rec = {}  # Dictionary for duplicate records