U++ forum: Welcome to the forum

static int cachesize=999; INITBLOCK{ #ifdef COMPILER_MSC int cpuInfo[4]; Zero(cpuInfo); __cpuid(cpuInfo, 0x80000006); #else unsigned int cpuInfo[4]; Zero(cpuInfo); __get_cpuid(0x80000006, &cpuInfo[0], &cpuInfo[1], &cpuInfo[2], &cpuInfo[3]); #endif cachesize=1024*(cpuInfo[2]>>16)*(cpuInfo[2]&0xff); }; void inline Fill3T(void *b, dword data, int len){ switch(len){ case 3: ((dword *)b)[2] = data; case 2: ((dword *)b)[1] = data; case 1: ((dword *)b)[0] = data; case 0: return; } __m128i q = _mm_set1_epi32(*(int*)&data); __m128i *w = (__m128i*)b; if(len >= 32) { __m128i *e = (__m128i*)b + (len>>2) - 8; if(len >= (cachesize>>2) && ((uintptr_t)w & 3) == 0) { // for really huge data, bypass the cache _mm_storeu_si128(w, q); // Head align int s=(-((int)((uintptr_t)b)>>2))&0x3; w = (__m128i*) ((dword*)b) + s; do { _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); }while(w<=e); _mm_sfence(); } else do { _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); }while(w<=e); } if(len & 16) { _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); } if(len & 8) { _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); } if(len & 4) { _mm_storeu_si128(w, q); } _mm_storeu_si128((__m128i*) (((dword*)b) + len - 4), q); // Tail align }

never_inline void HugeFill(dword *t, dword c, int len) { __m128i val4 = _mm_set1_epi32(*(int*)&c); auto Set4S = [&](int at) { _mm_stream_si128((__m128i *)(t + at), val4); }; while((uintptr_t)t & 15) { // align to 16 bytes for SSE *t++ = c; len--; } while(len >= 16) { Set4S(0); Set4S(4); Set4S(8); Set4S(12); t += 16; len -= 16; } while(len--) *t++ = c; _mm_sfence(); } void Fill6(dword *t, dword c, int len) { if(len >= 4) { __m128i val4 = _mm_set1_epi32(*(int*)&c); auto Set4 = [&](int at) { _mm_storeu_si128((__m128i *)(t + at), val4); }; if(len > 4*1024*1024 / 4) { HugeFill(t, c, len); return; } while(len >= 16) { Set4(0); Set4(4); Set4(8); Set4(12); t += 16; len -= 16; } if(len & 8) { Set4(0); Set4(4); t += 8; } if(len & 4) { Set4(0); t += 4; } } if(len & 3) t[0] = t[(len & 2) >> 1] = t[(len & 2) & ((len & 1) << 1)] = c; }

void Fill7(dword *t, dword data, int len){
	switch(len) {
		case 3: t[2] = data;
		case 2: t[1] = data;
		case 1: t[0] = data;
		case 0: return;
	}

	__m128i val4 = _mm_set1_epi32(data);
	auto Set4 = [&](int at) { _mm_storeu_si128((__m128i *)(t + at), val4); };

	Set4(len - 4); // fill tail
	if(len >= 32) {
		if(len >= 1024*1024) { // for really huge data, bypass the cache
			HugeFill(t, data, len);
			return;
		}
		const dword *e = t + len - 32;
		do {
			Set4(0); Set4(4); Set4(8); Set4(12);
			Set4(16); Set4(20); Set4(24); Set4(28);
			t += 32;
		}
		while(t <= e);
	}
	if(len & 16) {
		Set4(0); Set4(4); Set4(8); Set4(12);
		t += 16;
	}
	if(len & 8) {
		Set4(0); Set4(4);
		t += 8;
	}
	if(len & 4)
		Set4(0);
}

void Fill7a(dword *t, dword data, int len){
	if(len < 4) {
		if(len & 2) {
			t[0] = t[1] = data;
			t += 2;
		}
		if(len & 1)
			t[0] = data;
		return;
	}

	__m128i val4 = _mm_set1_epi32(data);
	auto Set4 = [&](int at) { _mm_storeu_si128((__m128i *)(t + at), val4); };

	Set4(len - 4); // fill tail
	if(len >= 32) {
		if(len >= 1024*1024) { // for really huge data, bypass the cache
			HugeFill(t, data, len);
			return;
		}
		const dword *e = t + len - 32;
		do {
			Set4(0); Set4(4); Set4(8); Set4(12);
			Set4(16); Set4(20); Set4(24); Set4(28);
			t += 32;
		}
		while(t <= e);
	}
	if(len & 16) {
		Set4(0); Set4(4); Set4(8); Set4(12);
		t += 16;
	}
	if(len & 8) {
		Set4(0); Set4(4);
		t += 8;
	}
	if(len & 4)
		Set4(0);
}

void Fill8(dword *t, dword data, int len){
	switch(len) {
		case 3: t[2] = data;
		case 2: t[1] = data;
		case 1: t[0] = data;
		case 0: return;
	}

	__m128i val4 = _mm_set1_epi32(data);
	auto Set4 = [&](int at) { _mm_storeu_si128((__m128i *)(t + at), val4); };

	Set4(len - 4); // fill tail
	if(len >= 32) {
		if(len >= 1024*1024) { // for really huge data, bypass the cache
			HugeFill(t, data, len);
			return;
		}
		int cnt = len >> 5;
		do {
			Set4(0); Set4(4); Set4(8); Set4(12);
			len -= 32;
			Set4(16); Set4(20); Set4(24); Set4(28);
			t += 32;
		}
		while(len >= 32);
	}
	switch((len >> 2) & 7) {
	case 7: Set4(24);
	case 6: Set4(20);
	case 5: Set4(16);
	case 4: Set4(12);
	case 3: Set4(8);
	case 2: Set4(4);
	case 1: Set4(0);
	}
}

void inline Fill3T2(dword *b, dword data, int len){ if(len<4){ if(len&1) *b++ = data; if(len&2){ *b++ = data; *b++ = data; } return; } __m128i q = _mm_set1_epi32(*(int*)&data); __m128i *w = (__m128i*)b; if(len >= 32) { __m128i *e = (__m128i*)b + (len>>2) - 8; if(len > 4*1024*1024 / 4 && ((uintptr_t)w & 3) == 0) { // for really huge data, bypass the cache _mm_storeu_si128(w, q); // Head align int s=(-((int)((uintptr_t)b)>>2))&0x3; w = (__m128i*) (b + s); do { _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); }while(w<=e); _mm_sfence(); } else do { _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); }while(w<=e); } if(len & 16) { _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); } if(len & 8) { _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); } if(len & 4) { _mm_storeu_si128(w, q); } _mm_storeu_si128((__m128i*) (b + len - 4), q); // Tail align }

never_inline void FillStream(dword *b, dword data, int len){ while((uintptr_t)b & 15){ // Try to align *b++=data; len--; }; __m128i *w = (__m128i *)b; __m128i q = _mm_set1_epi32((int)data); if(len>=16){ __m128i *e = w + (len>>2) - 3; do{ _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); }while(w<e); } if(len & 8) { _mm_stream_si128(w++, q); _mm_stream_si128(w++, q); } if(len & 4) { _mm_stream_si128(w++, q); } _mm_sfence(); _mm_storeu_si128((__m128i*)(b + len - 4), q); // Tail align } void inline Fill3T3(dword *b, dword data, int len){ if(len<4){ if(len&1) *b++ = data; if(len&2){ *b++ = data; *b++ = data; } return; } __m128i *w = (__m128i *)b; __m128i q = _mm_set1_epi32((int)data); if(len >= 32) { if(len>1024*1024 && (((uintptr_t)b & 3)==0)){ FillStream(b,data,len); return; } __m128i *e = w + (len>>2) - 7; do{ _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); }while(w<e); } if(len & 16) { _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); } if(len & 8) { _mm_storeu_si128(w++, q); _mm_storeu_si128(w++, q); } if(len & 4) { _mm_storeu_si128(w++, q); } _mm_storeu_si128((__m128i*)(b + len - 4), q); // Tail align }