phpython Code

--- a/trunk/python/python.tokenize.php
+++ b/trunk/python/python.tokenize.php
@@ -20,7 +20,9 @@
 		$z = 0;
 		
 		$g = array();
-				
+		
+		$f = true;
+		
 		do {
 			$v = static::python_tokenize_newline($c);
 			if($v>0){
@@ -46,10 +48,11 @@
 
 			if ($n > 0) {
 				$o[] = new lang_python_token($l,$k,'NEWLINE');
-			}
+				$f = true;
+			} 
 			
 			$w = static::python_tokenize_whitespace($c);
-
+						
 			if ($w > 0) {
 				$s = substr($c, 0, $w);
 				$c = substr($c, $w);
@@ -57,7 +60,7 @@
 				$s = '';
 			}
 
-			if ($n > 0) {
+			if ($f) {
 				$e = static::python_tokenize_compute_space($s);
 				if ($e != $i) {				
 					if ($e > $i) {
@@ -70,7 +73,9 @@
 							if ($h==$e) {break;}
 							$o[] = new lang_python_token($l,$k,'DEDENT');
 						} while (count($g)>0);	
-						$g[] = $e;
+						if ($e>0) {
+							$g[] = $e;
+						}
 						$i = $e;
 					} else {
 						$o[] = new lang_python_token($l,$k,'FAIL');							
@@ -80,6 +85,8 @@
 						
 			$t = static::python_tokenize_next($l,$k,$c);				
 
+			$f = false;
+			
 			$o[] = $t;
 
 			$c = substr($c,$t->len);