fixes #65, getting cyrillic going

README.md

@@ -155,6 +155,7 @@ textract.fromUrl(url, config, function( error, text ) {})
 
 ### 1.2.0
 * [#66](https://github.com/dbashford/textract/issues/66). textract will no longer put the info text to stdout about the extractors not being available or installed correctly.  Instead, if you attempt to use a supported extractor that did not initialize correctly, you will get an updated error message indicating that the type is supported by textract but that external dependencies were not located. As part of this update, error messages were updated a bit to list both the type and the file.
+* [#65](https://github.com/dbashford/textract/issues/65). Fixed issue where for `.odt` and `.docx` files with varying non-Latin characters (ex: cyrillic) were being stripped entirely of their content.
 
 ### 1.1.2
 * [#63](https://github.com/dbashford/textract/pull/63). PR added support for CSV.

lib/util.js

@@ -41,7 +41,6 @@ var replaceTextChars = function(text) {
   return text.trim()
     .replace( SINGLE_QUOTES, "'" )
     .replace( DOUBLE_QUOTES, '"' )
-    .replace( NON_ASCII_CHARS, '' )
     .replace( MULTI_SPACES, ' ' );
 };
 

test/general_test.js

@@ -63,6 +63,16 @@ describe('textract', function() {
     });
   });
 
+  it('can handle cyrillic', function(done) {
+    var filePath = path.join(__dirname, 'files', 'cyrillic.docx');
+    fromFileWithPath(filePath, function( error, text ) {
+      expect(error).to.be.null;
+      expect(text).to.be.a('string');
+      expect(text.substring(0,100)).to.eql( "Актуальность диссертационного исследования определяется необходимостью развития методологического об" );
+      done();
+    });
+  });
+
   describe("with multi line files", function() {
     it('strips line breaks', function(done) {
       var filePath = path.join(__dirname, 'files', 'multi-line.txt');

test/util_test.js

@@ -5,7 +5,7 @@ describe('textract util', function() {
   it('should normalize text output', function() {
     var text = "    “” ‘’ ą  \n\n some text";
     var result = util.replaceTextChars(text);
-    expect(result).to.equal("\"\" '' \n\n some text");
+    expect(result).to.equal("\"\" '' ą \n\n some text");
   });
 
 });