Fix parsing of code without trailing newlines (#940)

When the input doesn't have a trailing newline, but the last line had
exactly the amount of bytes as the current indentation level, the
tokenizer didn't emit a fake newline, causing parse errors (the grammar
expects newlines to conform with the Python spec).

I don't see any reason for fake newlines to be omitted in these cases,
so this PR removes that condition from the tokenizer.

Reported in #930.

native/libcst/src/lib.rs

@@ -149,6 +149,22 @@ mod test {
         parse_module("def g(a, b): ...", None).expect("parse error");
     }
 
+    #[test]
+    fn test_single_statement_with_no_newline() {
+        for src in &[
+            "(\n \\\n)",
+            "(\n  \\\n)",
+            "(\n    '''\n''')",
+            "del _",
+            "if _:\n    '''\n)'''",
+            "if _:\n    ('''\n''')",
+            "if _:\n     '''\n  '''",
+            "if _:\n        '''\n    ''' ",
+        ] {
+            parse_module(src, None).unwrap_or_else(|e| panic!("'{}' doesn't parse: {}", src, e));
+        }
+    }
+
     #[test]
     fn bol_offset_first_line() {
         assert_eq!(0, bol_offset("hello", 1));

native/libcst/src/tokenizer/core/mod.rs

@@ -330,10 +330,7 @@ impl<'t> TokState<'t> {
             return match self.text_pos.peek() {
                 // Check for EOF now
                 None => {
-                    if self.missing_nl_before_eof
-                        && self.text_pos.byte_column_number() != self.bol_width
-                        && !self.blank_line
-                    {
+                    if self.missing_nl_before_eof && !self.blank_line {
                         self.at_bol = true;
                         self.missing_nl_before_eof = false;
                         Ok(TokType::Newline)

native/libcst/src/tokenizer/tests.rs

@@ -718,6 +718,19 @@ fn test_fake_newline() {
     );
 }
 
+#[test]
+fn test_fake_newline_when_at_bol() {
+    assert_eq!(
+        tokenize_with_end_marker("(\n \\\n)", &default_config()),
+        Ok(vec![
+            (TokType::Op, "("),
+            (TokType::Op, ")"),
+            (TokType::Newline, ""),
+            (TokType::EndMarker, "")
+        ])
+    )
+}
+
 #[test]
 fn test_no_fake_newline_for_empty_input() {
     assert_eq!(