Scribd Logo
Upload

Welcome to Scribd!

  • 86 views

    Source Code For Program "Tokens". File "Tokens.c"

    Uploaded by

    Shirley Pérez
    This code defines functions for tokenizing text and counting the frequency of tokens. It uses a tree structure to store each unique token and its count. The install() function inserts a new token into the tree, incrementing the count if it already exists. The treeprint() function recursively traverses the tree and prints tokens whose count exceeds a minimum. The main() function handles command line arguments to configure options like case folding, then calls tokens() to process input and build the token frequency tree.

    Copyright:

    © All Rights Reserved
    Download as DOC, PDF, TXT or read online from Scribd

    Source Code For Program "Tokens". File "Tokens.c"

    Uploaded by

    Shirley Pérez
    86 views3 pages
    This code defines functions for tokenizing text and counting the frequency of tokens. It uses a tree structure to store each unique token and its count. The install() function inserts a new token into the tree, incrementing the count if it already exists. The treeprint() function recursively traverses the tree and prints tokens whose count exceeds a minimum. The main() function handles command line arguments to configure options like case folding, then calls tokens() to process input and build the token frequency tree.

    Original Description:

    token

    Original Title

    tokens

    Copyright

    © © All Rights Reserved

    Available Formats

    DOC, PDF, TXT or read online from Scribd

    Did you find this document useful?

    Is this content inappropriate?

    This code defines functions for tokenizing text and counting the frequency of tokens. It uses a tree structure to store each unique token and its count. The install() function inserts a new token into the tree, incrementing the count if it already exists. The treeprint() function recursively traverses the tree and prints tokens whose count exceeds a minimum. The main() function handles command line arguments to configure options like case folding, then calls tokens() to process input and build the token frequency tree.

    Copyright:

    © All Rights Reserved
    Download as DOC, PDF, TXT or read online from Scribd
    Download as doc, pdf, or txt
    86 views3 pages

    Source Code For Program "Tokens". File "Tokens.c"

    Uploaded by

    Shirley Pérez
    This code defines functions for tokenizing text and counting the frequency of tokens. It uses a tree structure to store each unique token and its count. The install() function inserts a new token into the tree, incrementing the count if it already exists. The treeprint() function recursively traverses the tree and prints tokens whose count exceeds a minimum. The main() function handles command line arguments to configure options like case folding, then calls tokens() to process input and build the token frequency tree.

    Copyright:

    © All Rights Reserved
    Download as DOC, PDF, TXT or read online from Scribd
    Download as doc, pdf, or txt
    You are on page 1of 3

    1

    2Source code for program “tokens”. File “tokens.c”


    3
    4 #include <stdio.h>
    5 #include <string.h>
    6 #include <assert.h>
    7
    8
    9 int Ignore = 0;
    10 int Mincount = 0;
    11 int Alpha = 0;
    12 char MapAllowed[256];
    13
    14
    15 typedef struct tnode
    16 {
    17 char *contents;
    18 int count;
    19 struct tnode *left;
    20 struct tnode *right;
    21 } TNODE;
    22
    23 void treeprint(tree) TNODE *tree;
    24 {
    25 if (tree != NULL) //Si tree es diferente a nulo, entra a la función y
    26 pasa a la linea 23
    27 {
    28 treeprint(tree->left);
    29 if (tree->count > Mincount)
    30 printf("%7d\t%s\n", tree->count, tree->contents);
    31 treeprint(tree->right);
    32 }
    33 }
    34
    35 TNODE *
    36 install(string, tree) char *string; TNODE * tree; //espera dos variables
    37 y retorna un nodo
    38 {
    39 int cond;
    40 assert(string != NULL);
    41 if (tree == NULL) //Si tree es diferente a nulo, entra a la función y
    42 pasa a la linea 42
    43
    44 {
    45 if (tree = (TNODE *) calloc(1, sizeof(TNODE)))
    46 {
    47 tree->contents = strdup(string);
    48 tree->count = 1;
    49 }
    50 }
    51 Else //Si tree es nulo, entra a la función y pasa a la linea 50
    52
    53 {
    54 cond = strcmp(string, tree->contents);

    1 3
    55 if (cond < 0) //Si cond es menor a 0 tree se dirige a la izquierda
    56 tree->left = install(string, tree->left);
    57 else if (cond == 0) //Si cond es igual a 0 tree aumenta
    58 tree->count++;
    59 else
    60 tree->right = install(string, tree->right);
    61 }
    62 return(tree); //La funcion retorna una variable nodo tree
    63 }
    64
    65 char *
    66 getword(ioptr) FILE *ioptr;
    67 {
    68 static char string[1024];
    69 char *ptr = string;
    70 register int c;
    71 assert(ioptr != NULL);
    72 for (;;)
    73 {
    74 c = getc(ioptr);
    75 if (c == EOF)
    76 if (ptr == string)
    77 return(NULL);
    78 else
    79 break;
    80 if (!MapAllowed[c])
    81 if (ptr == string)
    82 continue;
    83 else
    84 break;
    85 *ptr++ = MapAllowed[c];
    86 }
    87 *ptr = '\0';
    88 return(string);
    89 }
    90
    91 void tokens(ioptr) FILE *ioptr;
    92 //primero declara, entra al while y luego sale
    93 //Primero la declaracion, no ingresa al while y luego sale
    94 {
    95 TNODE *root = NULL;
    96 char *s;
    97 assert(ioptr != NULL);
    98 while (s = getword(ioptr)) //en el ciclo s se mantiene mientra que sea
    99 igual a getword(ioptr)
    100 root = install(s, root);
    101 treeprint(root);
    102 }
    103
    104 int main(argc, argv) int argc; char ** argv;
    105 {
    106 int c, errcnt = 0;
    107 extern char *optarg;
    108
    109 while ((c = getopt(argc, argv, "ac:im:")) != EOF)
    110 switch(c)
    111 {

    2 4
    112 case 'a': Alpha = 0; break;
    113 case 'c':
    114 while (*optarg)
    115 {
    116 MapAllowed[*optarg] = *optarg;
    117 optarg++;
    118 }
    119 break;
    120 case 'i': Ignore = 1; break;
    121 case 'm': Mincount = atoi(optarg); break;
    122 default: errcnt++;
    123 }
    124 if (errcnt)
    125 {
    126 fprintf(stderr, "Usage: %s [-i] [-c chars] [-m count]\n", *argv);
    127 return(1);
    128 }
    129 for (c = 'a'; c <= 'z'; c++)
    130 MapAllowed[c] = c;
    131 for (c = 'A'; c <= 'Z'; c++)
    132 MapAllowed[c] = Ignore ? c - 'A' + 'a' : c;
    133 if (!Alpha)
    134 for (c = '0'; c <= '9'; c++)
    135 MapAllowed[c] = c;
    136 tokens(stdin);
    137 return(0);
    138 }

    3 5

    You might also like