diff --git a/src/libpmem/libpmem.vcxproj b/src/libpmem/libpmem.vcxproj index 9612994775093f9221f53593c1d547e3ff0581a8..62faffcca5c4df47404e245118ff39e041d09432 100644 --- a/src/libpmem/libpmem.vcxproj +++ b/src/libpmem/libpmem.vcxproj @@ -42,6 +42,7 @@ <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_nt_sse2_clflushopt.c" /> <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_nt_sse2_clwb.c" /> <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_nt_sse2_empty.c" /> + <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_nt_sse2_noflush.c" /> <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_nt_avx_clflush.c"> <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> @@ -58,10 +59,15 @@ <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> </ClCompile> + <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_nt_avx_noflush.c"> + <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> + <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> + </ClCompile> <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_t_sse2_clflush.c" /> <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_t_sse2_clflushopt.c" /> <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_t_sse2_clwb.c" /> <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_t_sse2_empty.c" /> + <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_t_sse2_noflush.c" /> <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_t_avx_clflush.c"> <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> @@ -78,10 +84,15 @@ <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> </ClCompile> + <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_t_avx_noflush.c"> + <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> + <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> + </ClCompile> <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_nt_sse2_clflush.c" /> <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_nt_sse2_clflushopt.c" /> <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_nt_sse2_clwb.c" /> <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_nt_sse2_empty.c" /> + <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_nt_sse2_noflush.c" /> <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_nt_avx_clflush.c"> <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> @@ -98,10 +109,15 @@ <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> </ClCompile> + <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_nt_avx_noflush.c"> + <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> + <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> + </ClCompile> <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_t_sse2_clflush.c" /> <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_t_sse2_clflushopt.c" /> <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_t_sse2_clwb.c" /> <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_t_sse2_empty.c" /> + <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_t_sse2_noflush.c" /> <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_t_avx_clflush.c"> <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> @@ -118,6 +134,10 @@ <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> </ClCompile> + <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_t_avx_noflush.c"> + <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> + <EnableEnhancedInstructionSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AdvancedVectorExtensions</EnableEnhancedInstructionSet> + </ClCompile> </ItemGroup> <ItemGroup> <ClInclude Include="..\..\src\common\out.h" /> diff --git a/src/libpmem/libpmem.vcxproj.filters b/src/libpmem/libpmem.vcxproj.filters index 2038c434be9ced172e97266828da2ab099a7dab0..cc0a4f4d530e8d494659fcb609fdac66d537dbe1 100644 --- a/src/libpmem/libpmem.vcxproj.filters +++ b/src/libpmem/libpmem.vcxproj.filters @@ -101,6 +101,9 @@ <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_nt_sse2_empty.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_nt_sse2_noflush.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_nt_avx_clflush.c"> <Filter>Source Files</Filter> </ClCompile> @@ -113,6 +116,9 @@ <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_nt_avx_empty.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_nt_avx_noflush.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_t_sse2_clflush.c"> <Filter>Source Files</Filter> </ClCompile> @@ -125,6 +131,9 @@ <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_t_sse2_empty.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_t_sse2_noflush.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_t_avx_clflush.c"> <Filter>Source Files</Filter> </ClCompile> @@ -137,6 +146,9 @@ <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_t_avx_empty.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\..\src\libpmem\x86_64\memcpy\memcpy_t_avx_noflush.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_nt_sse2_clflush.c"> <Filter>Source Files</Filter> </ClCompile> @@ -149,6 +161,9 @@ <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_nt_sse2_empty.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_nt_sse2_noflush.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_nt_avx_clflush.c"> <Filter>Source Files</Filter> </ClCompile> @@ -161,6 +176,9 @@ <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_nt_avx_empty.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_nt_avx_noflush.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_t_sse2_clflush.c"> <Filter>Source Files</Filter> </ClCompile> @@ -173,6 +191,9 @@ <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_t_sse2_empty.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_t_sse2_noflush.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_t_avx_clflush.c"> <Filter>Source Files</Filter> </ClCompile> @@ -185,6 +206,9 @@ <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_t_avx_empty.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\..\src\libpmem\x86_64\memset\memset_t_avx_noflush.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\common\os_auto_flush_windows.c"> <Filter>Source Files</Filter> </ClCompile> diff --git a/src/libpmem/pmem.h b/src/libpmem/pmem.h index 3a5374e998e77b741d3242e83a439453a7854d12..7fa8ea2a5e3502481aec4c4e40cb789c3ffaf79a 100644 --- a/src/libpmem/pmem.h +++ b/src/libpmem/pmem.h @@ -1,5 +1,5 @@ /* - * Copyright 2014-2018, Intel Corporation + * Copyright 2014-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -39,6 +39,7 @@ #include <stddef.h> #include "libpmem.h" #include "util.h" +#include "valgrind_internal.h" #ifdef __cplusplus extern "C" { @@ -78,7 +79,8 @@ void *pmem_map_register(int fd, size_t len, const char *path, int is_dev_dax); static force_inline void flush_empty_nolog(const void *addr, size_t len) { - /* NOP */ + /* NOP, but tell pmemcheck about it */ + VALGRIND_DO_FLUSH(addr, len); } /* @@ -87,6 +89,8 @@ flush_empty_nolog(const void *addr, size_t len) static force_inline void flush64b_empty(const char *addr) { + /* NOP, but tell pmemcheck about it */ + VALGRIND_DO_FLUSH(addr, 64); } /* diff --git a/src/libpmem/x86_64/flags.inc b/src/libpmem/x86_64/flags.inc index 42d56c251abf9403636e7cd5223981b629a6381d..e4a0b077db999b16edbc80325615086baad8831f 100644 --- a/src/libpmem/x86_64/flags.inc +++ b/src/libpmem/x86_64/flags.inc @@ -1,4 +1,4 @@ -# Copyright 2018, Intel Corporation +# Copyright 2018-2020, Intel Corporation # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -41,35 +41,43 @@ $(objdir)/memcpy_nt_avx512f_clflush.o: CFLAGS += -mavx512f $(objdir)/memcpy_nt_avx512f_clflushopt.o: CFLAGS += -mavx512f $(objdir)/memcpy_nt_avx512f_clwb.o: CFLAGS += -mavx512f $(objdir)/memcpy_nt_avx512f_empty.o: CFLAGS += -mavx512f +$(objdir)/memcpy_nt_avx512f_noflush.o: CFLAGS += -mavx512f $(objdir)/memset_nt_avx512f_clflush.o: CFLAGS += -mavx512f $(objdir)/memset_nt_avx512f_clflushopt.o: CFLAGS += -mavx512f $(objdir)/memset_nt_avx512f_clwb.o: CFLAGS += -mavx512f $(objdir)/memset_nt_avx512f_empty.o: CFLAGS += -mavx512f +$(objdir)/memset_nt_avx512f_noflush.o: CFLAGS += -mavx512f $(objdir)/memcpy_nt_avx_clflush.o: CFLAGS += -mavx $(objdir)/memcpy_nt_avx_clflushopt.o: CFLAGS += -mavx $(objdir)/memcpy_nt_avx_clwb.o: CFLAGS += -mavx $(objdir)/memcpy_nt_avx_empty.o: CFLAGS += -mavx +$(objdir)/memcpy_nt_avx_noflush.o: CFLAGS += -mavx $(objdir)/memset_nt_avx_clflush.o: CFLAGS += -mavx $(objdir)/memset_nt_avx_clflushopt.o: CFLAGS += -mavx $(objdir)/memset_nt_avx_clwb.o: CFLAGS += -mavx $(objdir)/memset_nt_avx_empty.o: CFLAGS += -mavx +$(objdir)/memset_nt_avx_noflush.o: CFLAGS += -mavx $(objdir)/memcpy_t_avx512f_clflush.o: CFLAGS += -mavx512f $(objdir)/memcpy_t_avx512f_clflushopt.o: CFLAGS += -mavx512f $(objdir)/memcpy_t_avx512f_clwb.o: CFLAGS += -mavx512f $(objdir)/memcpy_t_avx512f_empty.o: CFLAGS += -mavx512f +$(objdir)/memcpy_t_avx512f_noflush.o: CFLAGS += -mavx512f $(objdir)/memset_t_avx512f_clflush.o: CFLAGS += -mavx512f $(objdir)/memset_t_avx512f_clflushopt.o: CFLAGS += -mavx512f $(objdir)/memset_t_avx512f_clwb.o: CFLAGS += -mavx512f $(objdir)/memset_t_avx512f_empty.o: CFLAGS += -mavx512f +$(objdir)/memset_t_avx512f_noflush.o: CFLAGS += -mavx512f $(objdir)/memcpy_t_avx_clflush.o: CFLAGS += -mavx $(objdir)/memcpy_t_avx_clflushopt.o: CFLAGS += -mavx $(objdir)/memcpy_t_avx_clwb.o: CFLAGS += -mavx $(objdir)/memcpy_t_avx_empty.o: CFLAGS += -mavx +$(objdir)/memcpy_t_avx_noflush.o: CFLAGS += -mavx $(objdir)/memset_t_avx_clflush.o: CFLAGS += -mavx $(objdir)/memset_t_avx_clflushopt.o: CFLAGS += -mavx $(objdir)/memset_t_avx_clwb.o: CFLAGS += -mavx $(objdir)/memset_t_avx_empty.o: CFLAGS += -mavx +$(objdir)/memset_t_avx_noflush.o: CFLAGS += -mavx CFLAGS += -Ix86_64 diff --git a/src/libpmem/x86_64/init.c b/src/libpmem/x86_64/init.c index a1e34cf249998530b7ab1389224f2df637a15015..384fd52de0fe9e43428480333ffac0d9c8d2a977 100644 --- a/src/libpmem/x86_64/init.c +++ b/src/libpmem/x86_64/init.c @@ -1,5 +1,5 @@ /* - * Copyright 2014-2018, Intel Corporation + * Copyright 2014-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -125,7 +125,7 @@ memmove_nodrain_##isa##_##flush(void *dest, const void *src, size_t len, \ return dest;\ \ if (flags & PMEM_F_MEM_NOFLUSH) \ - memmove_mov_##isa##_empty(dest, src, len); \ + memmove_mov_##isa##_noflush(dest, src, len); \ else if (flags & PMEM_F_MEM_MOVNT)\ memmove_movnt_##isa ##_##flush(dest, src, len);\ else if (flags & PMEM_F_MEM_MOV)\ @@ -146,7 +146,7 @@ memset_nodrain_##isa##_##flush(void *dest, int c, size_t len, unsigned flags)\ return dest;\ \ if (flags & PMEM_F_MEM_NOFLUSH) \ - memset_mov_##isa##_empty(dest, c, len); \ + memset_mov_##isa##_noflush(dest, c, len); \ else if (flags & PMEM_F_MEM_MOVNT)\ memset_movnt_##isa##_##flush(dest, c, len);\ else if (flags & PMEM_F_MEM_MOV)\ diff --git a/src/libpmem/x86_64/memcpy/memcpy_nt_avx.h b/src/libpmem/x86_64/memcpy/memcpy_nt_avx.h index 238a15ce252eecce63bad66ddf730348ad450969..11c506872cb939b362066b8e3e59ca7f76def7d8 100644 --- a/src/libpmem/x86_64/memcpy/memcpy_nt_avx.h +++ b/src/libpmem/x86_64/memcpy/memcpy_nt_avx.h @@ -1,5 +1,5 @@ /* - * Copyright 2017-2018, Intel Corporation + * Copyright 2017-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -37,8 +37,8 @@ #include "pmem.h" #include "avx.h" #include "flush.h" -#include "memcpy_avx.h" #include "memcpy_memset.h" +#include "memcpy_avx.h" #include "valgrind_internal.h" static force_inline void diff --git a/src/libpmem/x86_64/memcpy/memcpy_nt_avx512f.h b/src/libpmem/x86_64/memcpy/memcpy_nt_avx512f.h index 333d4c57238ab7b5d1608cd9aafd24a18f54b6c2..e133c14f7f00b03243f3db2452f2d71a06d2991c 100644 --- a/src/libpmem/x86_64/memcpy/memcpy_nt_avx512f.h +++ b/src/libpmem/x86_64/memcpy/memcpy_nt_avx512f.h @@ -1,5 +1,5 @@ /* - * Copyright 2017-2018, Intel Corporation + * Copyright 2017-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -37,8 +37,8 @@ #include "pmem.h" #include "avx.h" #include "flush.h" -#include "memcpy_avx512f.h" #include "memcpy_memset.h" +#include "memcpy_avx512f.h" #include "libpmem.h" #include "valgrind_internal.h" diff --git a/src/libpmem/x86_64/memcpy/memcpy_nt_avx512f_noflush.c b/src/libpmem/x86_64/memcpy/memcpy_nt_avx512f_noflush.c new file mode 100644 index 0000000000000000000000000000000000000000..0e5e614580a6d18fd993f87aa41734baece626b3 --- /dev/null +++ b/src/libpmem/x86_64/memcpy/memcpy_nt_avx512f_noflush.c @@ -0,0 +1,36 @@ +/* + * Copyright 2017-2020, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#define flush noflush +#define EXPORTED_SYMBOL memmove_movnt_avx512f_noflush +#define maybe_barrier barrier_after_ntstores +#include "memcpy_nt_avx512f.h" diff --git a/src/libpmem/x86_64/memcpy/memcpy_nt_avx_noflush.c b/src/libpmem/x86_64/memcpy/memcpy_nt_avx_noflush.c new file mode 100644 index 0000000000000000000000000000000000000000..d6c8652732f830f23a8b6fd34a539d6b8741b698 --- /dev/null +++ b/src/libpmem/x86_64/memcpy/memcpy_nt_avx_noflush.c @@ -0,0 +1,36 @@ +/* + * Copyright 2017-2020, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#define flush noflush +#define EXPORTED_SYMBOL memmove_movnt_avx_noflush +#define maybe_barrier barrier_after_ntstores +#include "memcpy_nt_avx.h" diff --git a/src/libpmem/x86_64/memcpy/memcpy_nt_sse2_noflush.c b/src/libpmem/x86_64/memcpy/memcpy_nt_sse2_noflush.c new file mode 100644 index 0000000000000000000000000000000000000000..4765d9f6230bb90762fe5bfd9c52e4c8216408a3 --- /dev/null +++ b/src/libpmem/x86_64/memcpy/memcpy_nt_sse2_noflush.c @@ -0,0 +1,36 @@ +/* + * Copyright 2017-2020, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#define flush noflush +#define EXPORTED_SYMBOL memmove_movnt_sse2_noflush +#define maybe_barrier barrier_after_ntstores +#include "memcpy_nt_sse2.h" diff --git a/src/libpmem/x86_64/memcpy/memcpy_t_avx.h b/src/libpmem/x86_64/memcpy/memcpy_t_avx.h index fa6aa23c22c318548db1127323928caa69ecab60..e740f1a199781754803370a84430feeffd28b6df 100644 --- a/src/libpmem/x86_64/memcpy/memcpy_t_avx.h +++ b/src/libpmem/x86_64/memcpy/memcpy_t_avx.h @@ -1,5 +1,5 @@ /* - * Copyright 2017-2018, Intel Corporation + * Copyright 2017-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -37,8 +37,8 @@ #include "pmem.h" #include "avx.h" #include "flush.h" -#include "memcpy_avx.h" #include "memcpy_memset.h" +#include "memcpy_avx.h" static force_inline void memmove_mov8x64b(char *dest, const char *src) diff --git a/src/libpmem/x86_64/memcpy/memcpy_t_avx512f.h b/src/libpmem/x86_64/memcpy/memcpy_t_avx512f.h index 542bba6a66acb1773dcd852ac41fec952fd1cd06..0ce5da3a0b7ead78e401167242e0980ba8e0203f 100644 --- a/src/libpmem/x86_64/memcpy/memcpy_t_avx512f.h +++ b/src/libpmem/x86_64/memcpy/memcpy_t_avx512f.h @@ -1,5 +1,5 @@ /* - * Copyright 2017-2018, Intel Corporation + * Copyright 2017-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -37,8 +37,8 @@ #include "pmem.h" #include "avx.h" #include "flush.h" -#include "memcpy_avx512f.h" #include "memcpy_memset.h" +#include "memcpy_avx512f.h" static force_inline void memmove_mov32x64b(char *dest, const char *src) diff --git a/src/libpmem/x86_64/memcpy/memcpy_t_avx512f_noflush.c b/src/libpmem/x86_64/memcpy/memcpy_t_avx512f_noflush.c new file mode 100644 index 0000000000000000000000000000000000000000..03dd282263d475891b989de0158702c5d9b23723 --- /dev/null +++ b/src/libpmem/x86_64/memcpy/memcpy_t_avx512f_noflush.c @@ -0,0 +1,36 @@ +/* + * Copyright 2017-2020, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#define flush64b noflush64b +#define flush noflush +#define EXPORTED_SYMBOL memmove_mov_avx512f_noflush +#include "memcpy_t_avx512f.h" diff --git a/src/libpmem/x86_64/memcpy/memcpy_t_avx_noflush.c b/src/libpmem/x86_64/memcpy/memcpy_t_avx_noflush.c new file mode 100644 index 0000000000000000000000000000000000000000..79b820a248ed914677d60de34298c6dfd36d5cf7 --- /dev/null +++ b/src/libpmem/x86_64/memcpy/memcpy_t_avx_noflush.c @@ -0,0 +1,36 @@ +/* + * Copyright 2017-2020, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#define flush64b noflush64b +#define flush noflush +#define EXPORTED_SYMBOL memmove_mov_avx_noflush +#include "memcpy_t_avx.h" diff --git a/src/libpmem/x86_64/memcpy/memcpy_t_sse2_noflush.c b/src/libpmem/x86_64/memcpy/memcpy_t_sse2_noflush.c new file mode 100644 index 0000000000000000000000000000000000000000..15580e74fed73117e5cfa13a6f37cac3e7974dee --- /dev/null +++ b/src/libpmem/x86_64/memcpy/memcpy_t_sse2_noflush.c @@ -0,0 +1,36 @@ +/* + * Copyright 2017-2020, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#define flush64b noflush64b +#define flush noflush +#define EXPORTED_SYMBOL memmove_mov_sse2_noflush +#include "memcpy_t_sse2.h" diff --git a/src/libpmem/x86_64/memcpy_memset.h b/src/libpmem/x86_64/memcpy_memset.h index 45c647fab687ba32166145ce98c839c1c1adba8f..fb925cf54ae9564f083573ec62ff0d213c3a07a5 100644 --- a/src/libpmem/x86_64/memcpy_memset.h +++ b/src/libpmem/x86_64/memcpy_memset.h @@ -1,5 +1,5 @@ /* - * Copyright 2014-2018, Intel Corporation + * Copyright 2014-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -56,6 +56,18 @@ no_barrier_after_ntstores(void) */ } +static inline void +noflush(const void *addr, size_t len) +{ + /* NOP, not even pmemcheck annotation */ +} + +static inline void +noflush64b(const char *addr) +{ + /* NOP, not even pmemcheck annotation */ +} + #ifndef AVX512F_AVAILABLE /* XXX not supported in MSVC version we currently use */ #ifdef _MSC_VER @@ -78,18 +90,22 @@ void memmove_mov_sse2_clflush(char *dest, const char *src, size_t len); void memmove_mov_sse2_clflushopt(char *dest, const char *src, size_t len); void memmove_mov_sse2_clwb(char *dest, const char *src, size_t len); void memmove_mov_sse2_empty(char *dest, const char *src, size_t len); +void memmove_mov_sse2_noflush(char *dest, const char *src, size_t len); void memmove_movnt_sse2_clflush(char *dest, const char *src, size_t len); void memmove_movnt_sse2_clflushopt(char *dest, const char *src, size_t len); void memmove_movnt_sse2_clwb(char *dest, const char *src, size_t len); void memmove_movnt_sse2_empty(char *dest, const char *src, size_t len); +void memmove_movnt_sse2_noflush(char *dest, const char *src, size_t len); void memset_mov_sse2_clflush(char *dest, int c, size_t len); void memset_mov_sse2_clflushopt(char *dest, int c, size_t len); void memset_mov_sse2_clwb(char *dest, int c, size_t len); void memset_mov_sse2_empty(char *dest, int c, size_t len); +void memset_mov_sse2_noflush(char *dest, int c, size_t len); void memset_movnt_sse2_clflush(char *dest, int c, size_t len); void memset_movnt_sse2_clflushopt(char *dest, int c, size_t len); void memset_movnt_sse2_clwb(char *dest, int c, size_t len); void memset_movnt_sse2_empty(char *dest, int c, size_t len); +void memset_movnt_sse2_noflush(char *dest, int c, size_t len); #endif #if AVX_AVAILABLE @@ -97,18 +113,22 @@ void memmove_mov_avx_clflush(char *dest, const char *src, size_t len); void memmove_mov_avx_clflushopt(char *dest, const char *src, size_t len); void memmove_mov_avx_clwb(char *dest, const char *src, size_t len); void memmove_mov_avx_empty(char *dest, const char *src, size_t len); +void memmove_mov_avx_noflush(char *dest, const char *src, size_t len); void memmove_movnt_avx_clflush(char *dest, const char *src, size_t len); void memmove_movnt_avx_clflushopt(char *dest, const char *src, size_t len); void memmove_movnt_avx_clwb(char *dest, const char *src, size_t len); void memmove_movnt_avx_empty(char *dest, const char *src, size_t len); +void memmove_movnt_avx_noflush(char *dest, const char *src, size_t len); void memset_mov_avx_clflush(char *dest, int c, size_t len); void memset_mov_avx_clflushopt(char *dest, int c, size_t len); void memset_mov_avx_clwb(char *dest, int c, size_t len); void memset_mov_avx_empty(char *dest, int c, size_t len); +void memset_mov_avx_noflush(char *dest, int c, size_t len); void memset_movnt_avx_clflush(char *dest, int c, size_t len); void memset_movnt_avx_clflushopt(char *dest, int c, size_t len); void memset_movnt_avx_clwb(char *dest, int c, size_t len); void memset_movnt_avx_empty(char *dest, int c, size_t len); +void memset_movnt_avx_noflush(char *dest, int c, size_t len); #endif #if AVX512F_AVAILABLE @@ -116,18 +136,22 @@ void memmove_mov_avx512f_clflush(char *dest, const char *src, size_t len); void memmove_mov_avx512f_clflushopt(char *dest, const char *src, size_t len); void memmove_mov_avx512f_clwb(char *dest, const char *src, size_t len); void memmove_mov_avx512f_empty(char *dest, const char *src, size_t len); +void memmove_mov_avx512f_noflush(char *dest, const char *src, size_t len); void memmove_movnt_avx512f_clflush(char *dest, const char *src, size_t len); void memmove_movnt_avx512f_clflushopt(char *dest, const char *src, size_t len); void memmove_movnt_avx512f_clwb(char *dest, const char *src, size_t len); void memmove_movnt_avx512f_empty(char *dest, const char *src, size_t len); +void memmove_movnt_avx512f_noflush(char *dest, const char *src, size_t len); void memset_mov_avx512f_clflush(char *dest, int c, size_t len); void memset_mov_avx512f_clflushopt(char *dest, int c, size_t len); void memset_mov_avx512f_clwb(char *dest, int c, size_t len); void memset_mov_avx512f_empty(char *dest, int c, size_t len); +void memset_mov_avx512f_noflush(char *dest, int c, size_t len); void memset_movnt_avx512f_clflush(char *dest, int c, size_t len); void memset_movnt_avx512f_clflushopt(char *dest, int c, size_t len); void memset_movnt_avx512f_clwb(char *dest, int c, size_t len); void memset_movnt_avx512f_empty(char *dest, int c, size_t len); +void memset_movnt_avx512f_noflush(char *dest, int c, size_t len); #endif extern size_t Movnt_threshold; diff --git a/src/libpmem/x86_64/memset/memset_nt_avx.h b/src/libpmem/x86_64/memset/memset_nt_avx.h index 7c3aab3f5e65d4c174bcbe5c2edf1dac58a68704..2c0d5f1fdc88ac3cd2d8ac4fa824336c62deed21 100644 --- a/src/libpmem/x86_64/memset/memset_nt_avx.h +++ b/src/libpmem/x86_64/memset/memset_nt_avx.h @@ -1,5 +1,5 @@ /* - * Copyright 2017-2018, Intel Corporation + * Copyright 2017-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -38,8 +38,8 @@ #include "avx.h" #include "flush.h" #include "libpmem.h" -#include "memset_avx.h" #include "memcpy_memset.h" +#include "memset_avx.h" #include "out.h" #include "valgrind_internal.h" diff --git a/src/libpmem/x86_64/memset/memset_nt_avx512f_noflush.c b/src/libpmem/x86_64/memset/memset_nt_avx512f_noflush.c new file mode 100644 index 0000000000000000000000000000000000000000..15f6d209e54f883e86f37ef8374240f0672f4867 --- /dev/null +++ b/src/libpmem/x86_64/memset/memset_nt_avx512f_noflush.c @@ -0,0 +1,36 @@ +/* + * Copyright 2017-2020, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#define flush noflush +#define EXPORTED_SYMBOL memset_movnt_avx512f_noflush +#define maybe_barrier barrier_after_ntstores +#include "memset_nt_avx512f.h" diff --git a/src/libpmem/x86_64/memset/memset_nt_avx_noflush.c b/src/libpmem/x86_64/memset/memset_nt_avx_noflush.c new file mode 100644 index 0000000000000000000000000000000000000000..13ccda1f632cd2078337a049b88a637cad0c5b3f --- /dev/null +++ b/src/libpmem/x86_64/memset/memset_nt_avx_noflush.c @@ -0,0 +1,36 @@ +/* + * Copyright 2017-2020, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#define flush noflush +#define EXPORTED_SYMBOL memset_movnt_avx_noflush +#define maybe_barrier barrier_after_ntstores +#include "memset_nt_avx.h" diff --git a/src/libpmem/x86_64/memset/memset_nt_sse2_noflush.c b/src/libpmem/x86_64/memset/memset_nt_sse2_noflush.c new file mode 100644 index 0000000000000000000000000000000000000000..ee29860c17b16d8537562b00eaa28d1a22e96923 --- /dev/null +++ b/src/libpmem/x86_64/memset/memset_nt_sse2_noflush.c @@ -0,0 +1,36 @@ +/* + * Copyright 2017-2020, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#define flush noflush +#define EXPORTED_SYMBOL memset_movnt_sse2_noflush +#define maybe_barrier barrier_after_ntstores +#include "memset_nt_sse2.h" diff --git a/src/libpmem/x86_64/memset/memset_t_avx.h b/src/libpmem/x86_64/memset/memset_t_avx.h index ce78fccaf231234f4e9db1495a0b4fde681a48fa..a1bca3247d831ea826addc9f8f4324b510a24024 100644 --- a/src/libpmem/x86_64/memset/memset_t_avx.h +++ b/src/libpmem/x86_64/memset/memset_t_avx.h @@ -1,5 +1,5 @@ /* - * Copyright 2017-2018, Intel Corporation + * Copyright 2017-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -37,8 +37,8 @@ #include "pmem.h" #include "avx.h" #include "flush.h" -#include "memset_avx.h" #include "memcpy_memset.h" +#include "memset_avx.h" static force_inline void memset_mov8x64b(char *dest, __m256i ymm) diff --git a/src/libpmem/x86_64/memset/memset_t_avx512f.h b/src/libpmem/x86_64/memset/memset_t_avx512f.h index 9c3b47cfe27df30c5a40d026aa87dbddae19bfe9..2421cecedc0bf4aca290b5199b51954b6cb49941 100644 --- a/src/libpmem/x86_64/memset/memset_t_avx512f.h +++ b/src/libpmem/x86_64/memset/memset_t_avx512f.h @@ -1,5 +1,5 @@ /* - * Copyright 2017-2018, Intel Corporation + * Copyright 2017-2020, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -37,8 +37,8 @@ #include "pmem.h" #include "avx.h" #include "flush.h" -#include "memset_avx512f.h" #include "memcpy_memset.h" +#include "memset_avx512f.h" static force_inline void memset_mov32x64b(char *dest, __m512i zmm) diff --git a/src/libpmem/x86_64/memset/memset_t_avx512f_noflush.c b/src/libpmem/x86_64/memset/memset_t_avx512f_noflush.c new file mode 100644 index 0000000000000000000000000000000000000000..fb9c6a7f4050f938df8c27409205717999859872 --- /dev/null +++ b/src/libpmem/x86_64/memset/memset_t_avx512f_noflush.c @@ -0,0 +1,36 @@ +/* + * Copyright 2017-2020, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#define flush64b noflush64b +#define flush noflush +#define EXPORTED_SYMBOL memset_mov_avx512f_noflush +#include "memset_t_avx512f.h" diff --git a/src/libpmem/x86_64/memset/memset_t_avx_noflush.c b/src/libpmem/x86_64/memset/memset_t_avx_noflush.c new file mode 100644 index 0000000000000000000000000000000000000000..e6f620016340dd3c0510b3f0338f30b0104ce2c4 --- /dev/null +++ b/src/libpmem/x86_64/memset/memset_t_avx_noflush.c @@ -0,0 +1,36 @@ +/* + * Copyright 2017-2020, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#define flush64b noflush64b +#define flush noflush +#define EXPORTED_SYMBOL memset_mov_avx_noflush +#include "memset_t_avx.h" diff --git a/src/libpmem/x86_64/memset/memset_t_sse2_noflush.c b/src/libpmem/x86_64/memset/memset_t_sse2_noflush.c new file mode 100644 index 0000000000000000000000000000000000000000..c2d6ccb542d52332949d1be5a483436e26bcfb3f --- /dev/null +++ b/src/libpmem/x86_64/memset/memset_t_sse2_noflush.c @@ -0,0 +1,36 @@ +/* + * Copyright 2017-2020, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#define flush64b noflush64b +#define flush noflush +#define EXPORTED_SYMBOL memset_mov_sse2_noflush +#include "memset_t_sse2.h" diff --git a/src/libpmem/x86_64/sources.inc b/src/libpmem/x86_64/sources.inc index e1b1a2b70f7c233662abba562a6a92a43b810c80..98b278e3c77822351ef492ccef92a1cd91c9f4c7 100644 --- a/src/libpmem/x86_64/sources.inc +++ b/src/libpmem/x86_64/sources.inc @@ -1,4 +1,4 @@ -# Copyright 2018, Intel Corporation +# Copyright 2018-2020, Intel Corporation # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -38,34 +38,42 @@ LIBPMEM_ARCH_SOURCE = init.c\ memcpy_nt_avx_clflushopt.c\ memcpy_nt_avx_clwb.c\ memcpy_nt_avx_empty.c\ + memcpy_nt_avx_noflush.c\ memcpy_nt_sse2_clflush.c\ memcpy_nt_sse2_clflushopt.c\ memcpy_nt_sse2_clwb.c\ memcpy_nt_sse2_empty.c\ + memcpy_nt_sse2_noflush.c\ memset_nt_avx_clflush.c\ memset_nt_avx_clflushopt.c\ memset_nt_avx_clwb.c\ memset_nt_avx_empty.c\ + memset_nt_avx_noflush.c\ memset_nt_sse2_clflush.c\ memset_nt_sse2_clflushopt.c\ memset_nt_sse2_clwb.c\ memset_nt_sse2_empty.c\ + memset_nt_sse2_noflush.c\ memcpy_t_avx_clflush.c\ memcpy_t_avx_clflushopt.c\ memcpy_t_avx_clwb.c\ memcpy_t_avx_empty.c\ + memcpy_t_avx_noflush.c\ memcpy_t_sse2_clflush.c\ memcpy_t_sse2_clflushopt.c\ memcpy_t_sse2_clwb.c\ memcpy_t_sse2_empty.c\ + memcpy_t_sse2_noflush.c\ memset_t_avx_clflush.c\ memset_t_avx_clflushopt.c\ memset_t_avx_clwb.c\ memset_t_avx_empty.c\ + memset_t_avx_noflush.c\ memset_t_sse2_clflush.c\ memset_t_sse2_clflushopt.c\ memset_t_sse2_clwb.c\ - memset_t_sse2_empty.c + memset_t_sse2_empty.c\ + memset_t_sse2_noflush.c AVX512F_PROG="\#include <immintrin.h>\n\#include <stdint.h>\nint main(){ uint64_t v[8]; __m512i zmm0 = _mm512_loadu_si512((__m512i *)&v); return 0;}" AVX512F_AVAILABLE := $(shell printf $(AVX512F_PROG) |\ @@ -76,17 +84,21 @@ LIBPMEM_ARCH_SOURCE += memcpy_nt_avx512f_clflush.c\ memcpy_nt_avx512f_clflushopt.c\ memcpy_nt_avx512f_clwb.c\ memcpy_nt_avx512f_empty.c\ + memcpy_nt_avx512f_noflush.c\ memset_nt_avx512f_clflush.c\ memset_nt_avx512f_clflushopt.c\ memset_nt_avx512f_clwb.c\ memset_nt_avx512f_empty.c\ + memset_nt_avx512f_noflush.c\ memcpy_t_avx512f_clflush.c\ memcpy_t_avx512f_clflushopt.c\ memcpy_t_avx512f_clwb.c\ memcpy_t_avx512f_empty.c\ + memcpy_t_avx512f_noflush.c\ memset_t_avx512f_clflush.c\ memset_t_avx512f_clflushopt.c\ memset_t_avx512f_clwb.c\ - memset_t_avx512f_empty.c + memset_t_avx512f_empty.c\ + memset_t_avx512f_noflush.c endif