/*****************************************************
 * Robots.c -- parse robots.txt file for urls.       *
 *                                                   *
 * Author: stderr (stderr.dev@gmail.com)             *
 *****************************************************/

#include <stdio.h>

void usage(void)
{
  printf("./robots.txt <input file> <output file>\n");
}

int main(int argc, char *argv[])
{
  FILE *input_file;
  FILE *output_file;

  char line[256];
  char url[256];

  if (argc < 3) {
    usage();
    exit(1);
  }
  printf("[+] robots parsing program\n");
  printf("[+] Written by stderr.\n");
  printf("############################\n");
  input_file = fopen(argv[1],"r");
  if (input_file == 0) {
    printf("Error opening file.\n");
    exit(1);
  }
  output_file = fopen(argv[2],"w");
  if (output_file == 0) {
    printf("Error opening file.\n");
    exit(1);
  }

  printf("[+] Parsing %s\n",argv[1]);
  while (fgets(line,sizeof(line),input_file) != NULL) {
    if (sscanf(line,"Disallow: %s", &url) > 0) {
      fprintf(output_file,"%s\nlocal\n",url);
    }
  }
  printf("[+] Results written to %s\n",argv[2]);

  fclose(input_file);
  fclose(output_file);  
  return 0;
}