Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix the endianness issue in AIX while running the benchmark. #3345

Closed
wants to merge 16 commits into from
Closed

Fix the endianness issue in AIX while running the benchmark. #3345

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
2fa5993
Fix the endianness issue in AIX while running the benchmark.
KamathForAIX Apr 4, 2024
ca42db4
Merge branch 'main' into gh-3330
KamathForAIX Apr 4, 2024
a86c2fd
Attempt to Fix Windows Build break
KamathForAIX Apr 4, 2024
e0412c5
Change or to and
KamathForAIX Apr 4, 2024
0561f7d
Fix format issue
KamathForAIX Apr 4, 2024
aad8301
Move macros to faiss/impl/platform_macros.h to simplify logic
KamathForAIX Apr 8, 2024
be1a750
Fix Format issues
KamathForAIX Apr 8, 2024
db1a5eb
Merge test case fix due to endianness in AIX
KamathForAIX Apr 8, 2024
82eac21
fix Format issues
KamathForAIX Apr 8, 2024
391f931
fix Format issues
KamathForAIX Apr 8, 2024
553eba6
Add Swap2Bytes definition for little endian in Level2-inl.h
KamathForAIX Apr 8, 2024
be9f6ef
Make FAISS_BIG_ENDIAN definition only in platform-macros.h
KamathForAIX Apr 11, 2024
62caf79
Make FAISS_BIG_ENDIAN to not be defined as 0 or 1
KamathForAIX Apr 23, 2024
dbfd236
Fix Format issue
KamathForAIX Apr 23, 2024
1809905
Fix Format issue
KamathForAIX Apr 23, 2024
d3bc22e
Move isBigEndian to faiss::cppcontrib namespace
KamathForAIX Apr 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions contrib/vecs_io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import sys
import numpy as np

"""
Expand All @@ -13,6 +14,8 @@

def ivecs_read(fname):
a = np.fromfile(fname, dtype='int32')
if sys.byteorder == 'big':
a.byteswap(inplace=True)
d = a[0]
return a.reshape(-1, d + 1)[:, 1:].copy()

Expand Down
112 changes: 95 additions & 17 deletions faiss/cppcontrib/detail/UintReader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

#pragma once

#include <faiss/impl/platform_macros.h>
#include <cstdint>

namespace faiss {
Expand All @@ -31,7 +32,11 @@ struct Uint8Reader {
if (N_ELEMENTS > CPOS + 3) {
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
codes + ELEMENT_TO_READ * 4);
#ifdef FAISS_BIG_ENDIAN
return (code32) >> 24;
#else
return (code32 & 0x000000FF);
#endif
} else {
return codes[CPOS];
}
Expand All @@ -40,7 +45,11 @@ struct Uint8Reader {
if (N_ELEMENTS > CPOS + 2) {
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
codes + ELEMENT_TO_READ * 4);
#ifdef FAISS_BIG_ENDIAN
return (code32 & 0x00FF0000) >> 16;
#else
return (code32 & 0x0000FF00) >> 8;
#endif
} else {
return codes[CPOS];
}
Expand All @@ -49,7 +58,11 @@ struct Uint8Reader {
if (N_ELEMENTS > CPOS + 1) {
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
codes + ELEMENT_TO_READ * 4);
#ifdef FAISS_BIG_ENDIAN
return (code32 & 0x0000FF00) >> 8;
#else
return (code32 & 0x00FF0000) >> 16;
#endif
} else {
return codes[CPOS];
}
Expand All @@ -58,7 +71,11 @@ struct Uint8Reader {
if (N_ELEMENTS > CPOS) {
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
codes + ELEMENT_TO_READ * 4);
#ifdef FAISS_BIG_ENDIAN
return (code32 & 0x000000FF);
#else
return (code32) >> 24;
#endif
} else {
return codes[CPOS];
}
Expand Down Expand Up @@ -87,40 +104,61 @@ struct Uint10Reader {
switch (SUB_ELEMENT) {
case 0: {
if (N_ELEMENTS > CPOS + 2) {
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
codes + ELEMENT_TO_READ * 5);
#ifdef FAISS_BIG_ENDIAN
code32 = Swap4Bytes(code32);
#endif
return (code32 & 0b0000001111111111);
} else {
const uint16_t code16 = *reinterpret_cast<const uint16_t*>(
uint16_t code16 = *reinterpret_cast<const uint16_t*>(
codes + ELEMENT_TO_READ * 5 + 0);
#ifdef FAISS_BIG_ENDIAN
code16 = Swap2Bytes(code16);
#endif
return (code16 & 0b0000001111111111);
}
}
case 1: {
if (N_ELEMENTS > CPOS + 1) {
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
codes + ELEMENT_TO_READ * 5);
#ifdef FAISS_BIG_ENDIAN
code32 = Swap4Bytes(code32);
#endif
return (code32 & 0b000011111111110000000000) >> 10;
} else {
const uint16_t code16 = *reinterpret_cast<const uint16_t*>(
uint16_t code16 = *reinterpret_cast<const uint16_t*>(
codes + ELEMENT_TO_READ * 5 + 1);
#ifdef FAISS_BIG_ENDIAN
code16 = Swap2Bytes(code16);
#endif
return (code16 & 0b0000111111111100) >> 2;
}
}
case 2: {
if (N_ELEMENTS > CPOS) {
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
codes + ELEMENT_TO_READ * 5);
#ifdef FAISS_BIG_ENDIAN
code32 = Swap4Bytes(code32);
#endif
return (code32 & 0b00111111111100000000000000000000) >> 20;
} else {
const uint16_t code16 = *reinterpret_cast<const uint16_t*>(
uint16_t code16 = *reinterpret_cast<const uint16_t*>(
codes + ELEMENT_TO_READ * 5 + 2);
#ifdef FAISS_BIG_ENDIAN
code16 = Swap2Bytes(code16);
#endif
return (code16 & 0b0011111111110000) >> 4;
}
}
case 3: {
const uint16_t code16 = *reinterpret_cast<const uint16_t*>(
uint16_t code16 = *reinterpret_cast<const uint16_t*>(
codes + ELEMENT_TO_READ * 5 + 3);
#ifdef FAISS_BIG_ENDIAN
code16 = Swap2Bytes(code16);
#endif
return (code16 & 0b1111111111000000) >> 6;
}
}
Expand All @@ -147,45 +185,69 @@ struct Uint12Reader {
switch (SUB_ELEMENT) {
case 0: {
if (N_ELEMENTS > CPOS + 2) {
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
codes + ELEMENT_TO_READ * 6);
#ifdef FAISS_BIG_ENDIAN
code32 = Swap4Bytes(code32);
#endif
return (code32 & 0b0000111111111111);
} else {
const uint16_t code16 = *reinterpret_cast<const uint16_t*>(
uint16_t code16 = *reinterpret_cast<const uint16_t*>(
codes + ELEMENT_TO_READ * 6 + 0);
#ifdef FAISS_BIG_ENDIAN
code16 = Swap2Bytes(code16);
#endif
return (code16 & 0b0000111111111111);
}
}
case 1: {
if (N_ELEMENTS > CPOS + 1) {
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
codes + ELEMENT_TO_READ * 6);
#ifdef FAISS_BIG_ENDIAN
code32 = Swap4Bytes(code32);
#endif
return (code32 & 0b111111111111000000000000) >> 12;
} else {
const uint16_t code16 = *reinterpret_cast<const uint16_t*>(
uint16_t code16 = *reinterpret_cast<const uint16_t*>(
codes + ELEMENT_TO_READ * 6 + 1);
#ifdef FAISS_BIG_ENDIAN
code16 = Swap2Bytes(code16);
#endif
return (code16 & 0b1111111111110000) >> 4;
}
}
case 2: {
if (N_ELEMENTS > CPOS + 1) {
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
codes + ELEMENT_TO_READ * 6 + 2);
#ifdef FAISS_BIG_ENDIAN
code32 = Swap4Bytes(code32);
#endif
return (code32 & 0b000011111111111100000000) >> 8;
} else {
const uint16_t code16 = *reinterpret_cast<const uint16_t*>(
uint16_t code16 = *reinterpret_cast<const uint16_t*>(
codes + ELEMENT_TO_READ * 6 + 3);
#ifdef FAISS_BIG_ENDIAN
code16 = Swap2Bytes(code16);
#endif
return (code16 & 0b0000111111111111);
}
}
case 3: {
if (N_ELEMENTS > CPOS) {
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
codes + ELEMENT_TO_READ * 6 + 2);
#ifdef FAISS_BIG_ENDIAN
code32 = Swap4Bytes(code32);
#endif
return (code32 & 0b11111111111100000000000000000000) >> 20;
} else {
const uint16_t code16 = *reinterpret_cast<const uint16_t*>(
uint16_t code16 = *reinterpret_cast<const uint16_t*>(
codes + ELEMENT_TO_READ * 6 + 4);
#ifdef FAISS_BIG_ENDIAN
code16 = Swap2Bytes(code16);
#endif
return (code16 & 0b1111111111110000) >> 4;
}
}
Expand All @@ -208,23 +270,39 @@ struct Uint16Reader {
switch (SUB_ELEMENT) {
case 0: {
if (N_ELEMENTS > CPOS + 1) {
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
codes + ELEMENT_TO_READ * 4);
#ifdef FAISS_BIG_ENDIAN
code32 = Swap4Bytes(code32);
#endif
return (code32 & 0x0000FFFF);
} else {
const uint16_t* const __restrict codesFp16 =
reinterpret_cast<const uint16_t*>(codes);
#ifdef FAISS_BIG_ENDIAN
uint16_t rt = codesFp16[CPOS];
rt = Swap2Bytes(rt);
return rt;
#endif
return codesFp16[CPOS];
}
}
case 1: {
if (N_ELEMENTS > CPOS) {
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
codes + ELEMENT_TO_READ * 4);
#ifdef FAISS_BIG_ENDIAN
code32 = Swap4Bytes(code32);
#endif
return code32 >> 16;
} else {
const uint16_t* const __restrict codesFp16 =
reinterpret_cast<const uint16_t*>(codes);
#ifdef FAISS_BIG_ENDIAN
uint16_t rt = codesFp16[CPOS];
rt = Swap2Bytes(rt);
return rt;
#endif
return codesFp16[CPOS];
}
}
Expand Down
Loading